1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6 fmt,
7 net::SocketAddr,
8 path::{Path, PathBuf},
9 sync::{Arc, Mutex, RwLock},
10 time::{Duration, Instant as StdInstant},
11};
12
13use crate::scopes::admission::denied_capability;
14use serde::{Deserialize, Serialize};
15use subc_control::{
16 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
17 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
18 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
19 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
20 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
21 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
22 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
23};
24use subc_protocol::{
25 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
26 error_codes,
27 manifest::{
28 validate_hello_capability_grammar, validate_hello_confirm_operations,
29 validate_hello_display_name, validate_hello_event_declarations,
30 validate_hello_flow_operations, validate_hello_operation_deferral,
31 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
32 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
33 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
34 },
35 peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
36 scope::{
37 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
38 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
39 SCOPE_SYNC_OP,
40 },
41 session::{
42 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
43 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
44 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
45 ROLE_VERSIONS_FIELD,
46 },
47 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
48 Priority, RouteTarget, PROTOCOL_VERSION,
49};
50use tokio::time::{timeout_at, Instant};
51use tracing::{debug, info, instrument::WithSubscriber, warn};
52
53use crate::{
54 caller_identity::CallerIdentity,
55 capability_requirements::{
56 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
57 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
58 RequirementStatus, RuntimeModule,
59 },
60 child_roster::DaemonShutdownFlag,
61 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
62 forwarding::{
63 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
64 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
65 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
66 },
67 observability::{
68 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
69 },
70 privileged_prompt::{prompt_text, PrivilegedOperation},
71 provenance::{
72 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
73 },
74 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
75 router::{daemon_error_flags, RouteCtx, RouterError},
76 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
77 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
78 stderr_tail::{CaptureState, TailEntry},
79 supervise::{
80 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, ReservedHelloVerification,
81 SpawnSubscribeRefusal, SupervisorHandle, SwapHelloAdmission,
82 },
83 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
84};
85
86pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
92
93const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
94const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
95const CAP_PING_PONG: &str = "ping_pong_v1";
96const CAP_SESSION_ATTACH: &str = "session_attach_v1";
97const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
98
99const SUBC_CONTROL_OPS: &[&str] = &[
100 ops::DAEMON_SHUTDOWN_INTENT,
101 ops::ACTIVITY_RECORD,
102 ops::ACTIVITY_QUERY,
103 ops::ACTIVITY_SUBSCRIBE,
104 ops::SERVER_DESCRIBE,
105 ops::CATALOG_LIST,
106 ops::ROUTE_OPEN,
107 ops::ROUTE_POLL,
108 ops::ROUTE_CLOSING,
109 ops::ROUTE_CLOSED,
110 ops::SUPERVISOR_LIST,
111 ops::SUPERVISOR_RESTART,
112 ops::SUPERVISOR_SWAP,
113 ops::SUPERVISOR_RELOAD,
114 ops::SUPERVISOR_RESCAN,
115 ops::SUPERVISOR_RELEASE_RESERVED,
116 ops::SUPERVISOR_SET_ENABLED,
117 ops::SUPERVISOR_HEALTH_PROBE,
118 ops::SUPERVISOR_HEALTH,
119 ops::SUPERVISOR_STDERR_TAIL,
120 ops::SUPERVISOR_TERMINALS,
121 ops::SUPERVISOR_ROUTES,
122 ops::SUPERVISOR_PROVENANCE,
123 ops::SUPERVISOR_SPAWN_SNAPSHOT,
124 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
125];
126
127const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
134
135mod agent_commands;
136mod rescan_checked;
137use rescan_checked::{RescanMode, RescanPlan};
138
139const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
140 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
141 "supervisor.live_roots",
142 "supervisor.agent_commands",
143 SCOPE_SYNC_OP,
144 SCOPE_APPLY_OP,
145 SCOPE_DESCRIBE_OP,
146 subc_protocol::scope::ROUTE_AUTHORIZE_OP,
147 "operator.confirm",
148];
149
150const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
154
155const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
156
157pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
165
166pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
181
182pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
200
201const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
202const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
203
204fn reload_verdict(
205 configured: &Path,
206 spawned_from: Option<&Path>,
207 image: subc_control::RunningImageAgreement,
208) -> PendingReloadVerdict {
209 let path = match spawned_from {
210 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
211 ReloadPathAgreement::Match
212 }
213 Some(spawned_from) => ReloadPathAgreement::Mismatch {
214 configured: configured.to_path_buf(),
215 spawned_from: spawned_from.to_path_buf(),
216 },
217 None => ReloadPathAgreement::Unavailable {
218 reason: if matches!(
219 image,
220 subc_control::RunningImageAgreement::Unavailable {
221 reason: subc_control::RunningImageUnavailableReason::NotRunning
222 }
223 ) {
224 ReloadPathUnavailableReason::NotRunning
225 } else {
226 ReloadPathUnavailableReason::SpawnedPathUnavailable
227 },
228 },
229 };
230 PendingReloadVerdict::new(path, image)
231}
232
233fn launch_spec_agreement(
234 configured: &crate::supervise::ModuleSpec,
235 running: bool,
236 spawned: Option<&crate::supervise::LaunchSpecDigest>,
237) -> subc_control::LaunchSpecAgreement {
238 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
239 if !running {
240 return Agreement::Unavailable {
241 reason: Reason::NotRunning,
242 };
243 }
244 match spawned {
245 None => Agreement::Unavailable {
246 reason: Reason::SpawnedSpecUnavailable,
247 },
248 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
249 Agreement::Match
250 }
251 Some(_) => Agreement::Mismatch,
252 }
253}
254
255fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
256 if configured == spawned_from {
257 return true;
258 }
259 #[cfg(windows)]
260 {
261 match (
267 std::fs::canonicalize(configured),
268 std::fs::canonicalize(spawned_from),
269 ) {
270 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
271 _ => false,
272 }
273 }
274 #[cfg(not(windows))]
275 {
276 false
277 }
278}
279
280#[derive(Clone)]
281struct DaemonProvenanceFacts {
282 build: DaemonBuildProvenance,
283 pid: Option<u32>,
284 started_at_ms: Option<u64>,
285 start_clock: Option<crate::clock::StartClock>,
286 executable_path: Option<PathBuf>,
287 executable_identity: Option<SpawnedFileIdentity>,
288 process_start_time: Option<u64>,
289 probe: ExecutableIdentityProbe,
290}
291
292impl Default for DaemonProvenanceFacts {
293 fn default() -> Self {
294 Self {
295 build: DaemonBuildProvenance {
296 build_git_sha: None,
297 build_lock_digest: None,
298 },
299 pid: None,
300 started_at_ms: None,
301 start_clock: None,
302 executable_path: None,
303 executable_identity: None,
304 process_start_time: None,
305 probe: ExecutableIdentityProbe::default(),
306 }
307 }
308}
309
310#[derive(Debug, Clone)]
311struct SupervisorRescanContext {
312 supervisor: Supervisor,
313 config_path: PathBuf,
314 configured_port: Option<u16>,
315 storage_config: Option<crate::daemon_config::StorageConfig>,
316 admission_facts_carrier_module_id: Option<String>,
317 admission_facts_targets: Option<Vec<String>>,
318 scope_authority_owners: Vec<String>,
319 federated_scope_owner: String,
320 privileged_operator_prompt: PrivilegedOperatorPrompt,
321 generation: Arc<std::sync::atomic::AtomicU64>,
327}
328
329const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
350 "reloading",
351 "supervisor_not_live",
352 "registration_not_active",
353 "no_forwarding_connection",
354 "relay_send_failed",
355];
356
357#[derive(Clone)]
359pub struct ControlHandler {
360 registry: Arc<Registry>,
361 activity: Option<crate::activity::ActivityLedger>,
362 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
363 forwarding: Arc<ForwardingTable>,
364 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
365 supervisor: SupervisorHandle,
366 pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
367 agent_declarations: agent_commands::Queues,
368 list_admission: Arc<tokio::sync::Semaphore>,
369 subc_capabilities: Arc<[String]>,
370 route_bind_relay_timeout: Duration,
374 route_bind_relay_timeouts: BTreeMap<String, Duration>,
378 route_bind_breakers: RouteBindBreakers,
381 route_bind_concurrency: RouteBindConcurrency,
384 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
389 route_bind_breaker_threshold: u32,
391 route_bind_breaker_cooldown: Duration,
393 health_probe_timeout: Duration,
394 storage_config: Option<crate::daemon_config::StorageConfig>,
397 machine_id: Option<crate::machine_id::MachineId>,
401 admission_facts_carrier_module_id: Option<String>,
402 admission_facts_targets: Option<Vec<String>>,
403 scopes: Arc<RwLock<ScopeTable>>,
407 scope_deadline_added: Arc<tokio::sync::Notify>,
411 scope_authority_owners: Vec<String>,
414 federated_scope_owner: String,
415 privileged_operator_prompt: PrivilegedOperatorPrompt,
417 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
420 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
421 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
422 local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
423 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
424 pub(crate) operator_apps:
425 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
426 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
431 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
437 #[cfg(test)]
439 pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
440 #[cfg(test)]
441 pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
442 #[allow(dead_code)]
446 #[cfg(windows)]
447 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
448 #[allow(dead_code)]
449 #[cfg(windows)]
450 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
451 #[allow(dead_code)]
455 #[cfg(windows)]
456 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
457 #[cfg(all(test, windows))]
460 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
461 #[cfg(test)]
462 caller_lookup: Option<TestCallerLookup>,
463 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
466 rescan: Option<SupervisorRescanContext>,
467 connected_clients: ConnectedClients,
468 counters: DaemonCounters,
469 local_endpoint_unavailable: Option<String>,
472 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
473 daemon_shutdown: DaemonShutdownFlag,
478 daemon_provenance: DaemonProvenanceFacts,
479 #[cfg(test)]
480 control_dispatch_delay: Option<Duration>,
481 #[cfg(test)]
482 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
483 #[cfg(test)]
487 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
488 #[cfg(test)]
494 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
495}
496
497impl fmt::Debug for ControlHandler {
498 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
499 f.debug_struct("ControlHandler")
500 .field("registry", &self.registry)
501 .field("forwarding", &self.forwarding)
502 .field("process_liveness", &self.process_liveness.is_some())
503 .field("supervisor", &self.supervisor)
504 .field("subc_capabilities", &self.subc_capabilities)
505 .finish()
506 }
507}
508
509struct RouteOpenRequest {
510 target: RouteTarget,
511 identity: BindIdentity,
512 consumer_identity: Option<ConsumerIdentity>,
513 consumer_capabilities: Option<Vec<String>>,
514 role_versions: Option<BTreeMap<String, String>>,
515 busy_refusals: Option<bool>,
516 admission_facts: Option<serde_json::Value>,
517 scope: Option<ScopeSelector>,
518}
519
520#[derive(Clone, Copy)]
521enum RouteOpenPhase {
522 Admission,
523 ProjectRoot,
524 EgressReservation,
525 BindSend,
526 BindAckWait,
527}
528
529struct RouteOpenTimings {
533 module_id: String,
534 principal: String,
535 admission: Duration,
536 project_root: Duration,
537 egress_reservation: Duration,
538 bind_send: Duration,
539 bind_ack_wait: Duration,
540 phase: Option<RouteOpenPhase>,
541 phase_started_at: Instant,
542}
543
544impl RouteOpenTimings {
545 fn new() -> Self {
546 Self {
547 module_id: String::new(),
548 principal: "unresolved".to_string(),
550 admission: Duration::ZERO,
551 project_root: Duration::ZERO,
552 egress_reservation: Duration::ZERO,
553 bind_send: Duration::ZERO,
554 bind_ack_wait: Duration::ZERO,
555 phase: Some(RouteOpenPhase::Admission),
556 phase_started_at: Instant::now(),
557 }
558 }
559
560 fn finish(&mut self) {
561 let elapsed = self.phase_started_at.elapsed();
562 match self.phase.take() {
563 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
564 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
565 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
566 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
567 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
568 None => {}
569 }
570 }
571
572 fn enter(&mut self, phase: RouteOpenPhase) {
573 self.finish();
574 self.phase = Some(phase);
575 self.phase_started_at = Instant::now();
576 }
577}
578
579struct RouteBindReservationGuard {
580 forwarding: Arc<ForwardingTable>,
581 endpoint: ModuleEndpointId,
582 relay_corr: u64,
583 armed: bool,
584}
585
586struct ModuleControlRpcGuard {
587 forwarding: Arc<ForwardingTable>,
588 endpoint: ModuleEndpointId,
589 corr: u64,
590 armed: bool,
591}
592
593impl ModuleControlRpcGuard {
594 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
595 Self {
596 forwarding,
597 endpoint,
598 corr,
599 armed: true,
600 }
601 }
602
603 fn disarm(&mut self) {
604 self.armed = false;
605 }
606}
607
608impl Drop for ModuleControlRpcGuard {
609 fn drop(&mut self) {
610 if self.armed {
611 let _ = self
612 .forwarding
613 .cancel_module_control_rpc(self.endpoint, self.corr);
614 }
615 }
616}
617
618impl RouteBindReservationGuard {
619 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
620 Self {
621 forwarding,
622 endpoint,
623 relay_corr,
624 armed: true,
625 }
626 }
627
628 fn release_and_disarm(&mut self) {
629 if !self.armed {
630 return;
631 }
632 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
633 self.endpoint,
634 self.relay_corr,
635 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
636 ) {
637 send_goodbye_target_best_effort(
638 &self.forwarding.counters(),
639 &target,
640 "canceled route.bind",
641 );
642 }
643 self.armed = false;
644 }
645
646 fn disarm(&mut self) {
647 self.armed = false;
648 }
649}
650
651impl Drop for RouteBindReservationGuard {
652 fn drop(&mut self) {
653 self.release_and_disarm();
654 }
655}
656
657#[derive(Debug, Clone, Default)]
687pub(crate) struct RouteBindBreakers {
688 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
689}
690
691#[derive(Debug, Clone, Default)]
692pub(crate) struct RouteBindConcurrency {
693 modules: Arc<Mutex<HashMap<String, usize>>>,
694}
695
696struct RouteBindConcurrencyGuard {
697 concurrency: RouteBindConcurrency,
698 module_id: String,
699}
700
701impl RouteBindConcurrency {
702 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
705 let mut modules = self
706 .modules
707 .lock()
708 .expect("route.bind concurrency mutex poisoned");
709 let in_flight = modules.entry(module_id.to_string()).or_default();
710 if *in_flight >= limit {
711 return Err(*in_flight);
712 }
713 *in_flight += 1;
714 Ok(RouteBindConcurrencyGuard {
715 concurrency: self.clone(),
716 module_id: module_id.to_string(),
717 })
718 }
719}
720
721impl Drop for RouteBindConcurrencyGuard {
722 fn drop(&mut self) {
723 let mut modules = self
724 .concurrency
725 .modules
726 .lock()
727 .expect("route.bind concurrency mutex poisoned");
728 let remove = {
729 let in_flight = modules
730 .get_mut(&self.module_id)
731 .expect("admitted route.bind has a concurrency entry");
732 *in_flight -= 1;
733 *in_flight == 0
734 };
735 if remove {
736 modules.remove(&self.module_id);
737 }
738 }
739}
740
741#[derive(Debug, Default)]
742struct ModuleBreakerState {
743 consecutive_timeouts: u32,
745 cooldown_until: Option<Instant>,
748 probe_in_flight: Option<Arc<()>>,
753}
754
755enum RouteBindAdmission<'a> {
757 Admitted {
758 guard: RouteBindBreakerGuard<'a>,
759 probe: bool,
762 },
763 Refused {
764 consecutive_timeouts: u32,
765 retry_in: Duration,
769 probe_in_flight: bool,
770 },
771}
772
773struct RouteBindBreakerGuard<'a> {
780 breakers: RouteBindBreakers,
781 module_id: &'a str,
782 probe_token: Option<Arc<()>>,
783 settled: bool,
784}
785
786impl RouteBindBreakerGuard<'_> {
787 fn record_accepted(&mut self) -> bool {
791 self.settled = true;
792 self.breakers.record_accepted(self.module_id)
793 }
794
795 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
798 self.settled = true;
799 self.breakers.record_timeout(
800 self.module_id,
801 self.probe_token.as_ref(),
802 threshold,
803 cooldown,
804 )
805 }
806
807 fn record_inconclusive(&mut self) {
816 self.settled = true;
817 self.breakers
818 .record_inconclusive(self.module_id, self.probe_token.as_ref());
819 }
820}
821
822impl Drop for RouteBindBreakerGuard<'_> {
823 fn drop(&mut self) {
824 if !self.settled {
825 self.breakers
826 .record_inconclusive(self.module_id, self.probe_token.as_ref());
827 }
828 }
829}
830
831struct BreakerOpened {
835 consecutive_timeouts: u32,
836 reopened_after_probe: bool,
839}
840
841impl RouteBindBreakers {
842 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
843 self.modules
844 .lock()
845 .expect("route.bind breaker mutex poisoned")
846 }
847
848 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
851 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
852 probe: probe_token.is_some(),
853 guard: RouteBindBreakerGuard {
854 breakers: self.clone(),
855 module_id,
856 probe_token,
857 settled: false,
858 },
859 };
860
861 let mut modules = self.lock();
862 let Some(state) = modules.get_mut(module_id) else {
863 return admitted(None);
864 };
865 let Some(cooldown_until) = state.cooldown_until else {
866 return admitted(None);
867 };
868 if state.probe_in_flight.is_some() {
869 return RouteBindAdmission::Refused {
870 consecutive_timeouts: state.consecutive_timeouts,
871 retry_in: Duration::ZERO,
872 probe_in_flight: true,
873 };
874 }
875 let now = Instant::now();
876 if now < cooldown_until {
877 return RouteBindAdmission::Refused {
878 consecutive_timeouts: state.consecutive_timeouts,
879 retry_in: cooldown_until - now,
880 probe_in_flight: false,
881 };
882 }
883 let token = Arc::new(());
884 state.probe_in_flight = Some(Arc::clone(&token));
885 admitted(Some(token))
886 }
887
888 fn record_accepted(&self, module_id: &str) -> bool {
889 self.lock()
890 .remove(module_id)
891 .is_some_and(|state| state.cooldown_until.is_some())
892 }
893
894 fn record_timeout(
895 &self,
896 module_id: &str,
897 probe_token: Option<&Arc<()>>,
898 threshold: u32,
899 cooldown: Duration,
900 ) -> Option<BreakerOpened> {
901 let mut modules = self.lock();
902 let state = modules.entry(module_id.to_string()).or_default();
903 let was_open = state.cooldown_until.is_some();
904 let was_probe = Self::owns_probe(state, probe_token);
905 if was_probe {
906 state.probe_in_flight = None;
907 }
908 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
909 if state.consecutive_timeouts < threshold {
910 return None;
911 }
912 state.cooldown_until = Some(Instant::now() + cooldown);
913 Some(BreakerOpened {
914 consecutive_timeouts: state.consecutive_timeouts,
915 reopened_after_probe: was_open && was_probe,
916 })
917 }
918
919 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
920 state
924 .probe_in_flight
925 .as_ref()
926 .zip(token)
927 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
928 }
929
930 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
931 if let Some(state) = self.lock().get_mut(module_id) {
932 if Self::owns_probe(state, probe_token) {
933 state.probe_in_flight = None;
934 }
935 }
936 }
937
938 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
953 self.lock()
954 .remove(module_id)
955 .map(|state| state.consecutive_timeouts)
956 .filter(|discarded| *discarded > 0)
957 }
958
959 fn open_snapshot(&self) -> Option<serde_json::Value> {
966 let now = Instant::now();
967 let modules = self.lock();
968 let open = modules
969 .iter()
970 .filter_map(|(module_id, state)| {
971 let cooldown_until = state.cooldown_until?;
972 Some((
973 module_id.clone(),
974 serde_json::json!({
975 "consecutive_timeouts": state.consecutive_timeouts,
976 "cooldown_remaining_ms":
977 cooldown_until.saturating_duration_since(now).as_millis() as u64,
978 "probe_in_flight": state.probe_in_flight.is_some(),
979 }),
980 ))
981 })
982 .collect::<serde_json::Map<String, serde_json::Value>>();
983 (!open.is_empty()).then_some(serde_json::Value::Object(open))
984 }
985}
986
987impl ControlHandler {
988 pub fn new(registry: Arc<Registry>) -> Self {
989 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
990 }
991
992 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
993 let counters = forwarding.counters();
994 let route_bind_breakers = forwarding.route_bind_breakers();
998 let route_bind_concurrency = forwarding.route_bind_concurrency();
999 let route_outages = forwarding.route_outages();
1000 Self {
1001 registry,
1002 activity: None,
1003 activity_streams: Arc::new(Mutex::new(HashMap::new())),
1004 forwarding,
1005 process_liveness: None,
1006 supervisor: SupervisorHandle::new(),
1007 resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1008 env!("CARGO_PKG_VERSION").to_owned(),
1009 )),
1010 agent_declarations: agent_commands::Queues::default(),
1011 list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1012 subc_capabilities: Arc::from([
1013 CAP_MANIFEST_REGISTRATION.to_string(),
1014 CAP_CHANNEL_LIFECYCLE.to_string(),
1015 CAP_PING_PONG.to_string(),
1016 CAP_SESSION_ATTACH.to_string(),
1017 CAP_ADMISSION_FACTS_RELAY.to_string(),
1018 CAP_SCOPES_V1.to_string(),
1019 subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY.to_string(),
1020 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1021 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1022 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1023 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1024 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1025 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1026 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1027 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1028 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1029 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1030 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1031 ]),
1032 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1033 route_bind_relay_timeouts: BTreeMap::new(),
1034 route_bind_breakers,
1035 route_bind_concurrency,
1036 route_outages,
1037 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1038 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1039 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1040 storage_config: None,
1041 machine_id: None,
1042 admission_facts_carrier_module_id: None,
1043 admission_facts_targets: None,
1044 scopes: Arc::new(RwLock::new(ScopeTable::new(
1045 crate::daemon_config::default_scope_authority_owners(),
1046 ))),
1047 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1048 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1049 federated_scope_owner: crate::daemon_config::default_federated_scope_owner(),
1050 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1051 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1052 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1053 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1054 local_connections: Arc::new(Mutex::new(HashSet::new())),
1055 operator_app_policy: Default::default(),
1056 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1057 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1058 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1059 #[cfg(test)]
1060 app_open_test_platform: None,
1061 #[cfg(test)]
1062 app_open_test_launch: None,
1063 #[cfg(windows)]
1064 windows_verify: None,
1065 #[cfg(windows)]
1066 windows_flash: None,
1067 #[cfg(windows)]
1068 windows_track: None,
1069 #[cfg(all(test, windows))]
1070 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1071 #[cfg(test)]
1072 caller_lookup: None,
1073 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1074 rescan: None,
1075 connected_clients: ConnectedClients::new(),
1076 local_endpoint_unavailable: None,
1077 counters,
1078 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1079 daemon_shutdown: DaemonShutdownFlag::default(),
1080 daemon_provenance: DaemonProvenanceFacts::default(),
1081 #[cfg(test)]
1082 control_dispatch_delay: None,
1083 #[cfg(test)]
1084 provenance_probe_override: None,
1085 #[cfg(test)]
1086 route_bind_relay_sent_hold: None,
1087 #[cfg(test)]
1088 hello_registry_to_forwarding_hold: None,
1089 }
1090 }
1091
1092 pub fn with_storage_config(
1095 mut self,
1096 storage_config: Option<crate::daemon_config::StorageConfig>,
1097 ) -> Self {
1098 self.storage_config = storage_config;
1099 self
1100 }
1101
1102 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1105 self.activity = Some(crate::activity::ActivityLedger::open(
1106 run_dir.join("activity.jsonl"),
1107 incarnation,
1108 ));
1109 self.supervisor.set_activity(self.activity.clone());
1110 self
1111 }
1112
1113 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1116 self.machine_id = machine_id;
1117 self
1118 }
1119
1120 pub fn with_admission_facts_config(
1124 mut self,
1125 carrier_module_id: Option<String>,
1126 targets: Option<Vec<String>>,
1127 ) -> Self {
1128 self.admission_facts_carrier_module_id = carrier_module_id;
1129 self.admission_facts_targets = targets;
1130 self
1131 }
1132
1133 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1137 self.scopes = Arc::new(RwLock::new(
1138 ScopeTable::new(owners.iter().cloned())
1139 .with_federated_owner(self.federated_scope_owner.clone()),
1140 ));
1141 self.scope_authority_owners = owners;
1142 self
1143 }
1144
1145 pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
1147 self.scopes = Arc::new(RwLock::new(
1148 ScopeTable::new(self.scope_authority_owners.iter().cloned())
1149 .with_federated_owner(owner.clone()),
1150 ));
1151 self.federated_scope_owner = owner;
1152 self
1153 }
1154
1155 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1157 self.privileged_operator_prompt = mode;
1158 self
1159 }
1160
1161 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1164 self.route_bind_relay_timeout = timeout;
1165 self
1166 }
1167
1168 pub fn with_route_bind_relay_timeouts(
1174 mut self,
1175 timeouts: impl IntoIterator<Item = (String, Duration)>,
1176 ) -> Self {
1177 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1178 self
1179 }
1180
1181 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1187 self.route_bind_relay_timeouts
1188 .get(module_id)
1189 .copied()
1190 .unwrap_or(self.route_bind_relay_timeout)
1191 }
1192
1193 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1201 self.route_bind_breaker_threshold = threshold.max(1);
1202 self.route_bind_breaker_cooldown = cooldown;
1203 self
1204 }
1205
1206 #[cfg(test)]
1207 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1208 self.health_probe_timeout = timeout;
1209 self
1210 }
1211
1212 #[cfg(test)]
1213 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1214 self.control_dispatch_delay = Some(delay);
1215 self
1216 }
1217
1218 #[cfg(test)]
1222 pub(crate) fn with_route_bind_relay_sent_hold(
1223 mut self,
1224 hold: Arc<tokio::sync::Barrier>,
1225 ) -> Self {
1226 self.route_bind_relay_sent_hold = Some(hold);
1227 self
1228 }
1229
1230 #[cfg(test)]
1235 pub(crate) fn with_hello_registry_to_forwarding_hold(
1236 mut self,
1237 hold: Arc<std::sync::Barrier>,
1238 ) -> Self {
1239 self.hello_registry_to_forwarding_hold = Some(hold);
1240 self
1241 }
1242
1243 #[cfg(test)]
1244 fn hold_between_registry_and_forwarding(&self) {
1245 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1246 hold.wait();
1247 hold.wait();
1248 }
1249 }
1250
1251 pub fn with_process_liveness(
1252 mut self,
1253 process_liveness: Arc<dyn ModuleProcessLiveness>,
1254 ) -> Self {
1255 self.process_liveness = Some(process_liveness);
1256 self
1257 }
1258
1259 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1260 self.supervisor = supervisor;
1261 self.supervisor.set_activity(self.activity.clone());
1262 self
1263 }
1264
1265 pub fn with_daemon_provenance(
1266 mut self,
1267 pid: u32,
1268 started_at_ms: u64,
1269 executable_path: Option<PathBuf>,
1270 build_git_sha: Option<String>,
1271 build_lock_digest: Option<String>,
1272 ) -> Self {
1273 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1274 let process_start_time = process_start_time(pid);
1275 self.daemon_provenance = DaemonProvenanceFacts {
1276 build: DaemonBuildProvenance {
1277 build_git_sha,
1278 build_lock_digest,
1279 },
1280 pid: Some(pid),
1281 started_at_ms: Some(started_at_ms),
1282 start_clock: None,
1283 executable_path,
1284 executable_identity,
1285 process_start_time,
1286 probe: ExecutableIdentityProbe::default(),
1287 };
1288 self
1289 }
1290
1291 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1292 self.daemon_provenance.start_clock = Some(clock);
1293 self
1294 }
1295
1296 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1298 self.scopes
1299 .write()
1300 .unwrap_or_else(|poisoned| poisoned.into_inner())
1301 .set_wall_clock(Arc::new(clock));
1302 self
1303 }
1304
1305 #[cfg(test)]
1306 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1307 self.provenance_probe_override = Some(result);
1308 self
1309 }
1310
1311 pub fn with_capability_config(
1315 self,
1316 modules: impl IntoIterator<Item = (String, bool)>,
1317 reserved_capabilities: BTreeMap<String, String>,
1318 ) -> Self {
1319 self.capability_evaluator
1320 .configure(modules, reserved_capabilities);
1321 self
1322 }
1323
1324 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1328 self.daemon_shutdown = flag;
1329 self
1330 }
1331
1332 pub fn with_supervisor_rescan(
1333 mut self,
1334 supervisor: Supervisor,
1335 config_path: impl Into<PathBuf>,
1336 configured_port: Option<u16>,
1337 ) -> Self {
1338 self.rescan = Some(SupervisorRescanContext {
1339 supervisor,
1340 config_path: config_path.into(),
1341 configured_port,
1342 storage_config: self.storage_config.clone(),
1343 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1344 admission_facts_targets: self.admission_facts_targets.clone(),
1345 scope_authority_owners: self.scope_authority_owners.clone(),
1346 federated_scope_owner: self.federated_scope_owner.clone(),
1347 privileged_operator_prompt: self.privileged_operator_prompt,
1348 generation: Arc::default(),
1349 });
1350 self
1351 }
1352
1353 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1354 self.connected_clients = connected_clients;
1355 self
1356 }
1357
1358 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1361 self.local_endpoint_unavailable = reason;
1362 self
1363 }
1364
1365 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1366 Arc::clone(&self.forwarding)
1367 }
1368
1369 pub(crate) fn counters(&self) -> DaemonCounters {
1370 self.counters.clone()
1371 }
1372
1373 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1376 tokio::spawn(async move {
1377 loop {
1378 self.capability_evaluator
1379 .wait_for_change_or_deadline()
1380 .await;
1381 self.refresh_capability_requirements();
1382 }
1383 });
1384 }
1385
1386 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1392 tokio::spawn(async move {
1393 loop {
1394 let has_deadlines = self
1395 .scopes
1396 .read()
1397 .unwrap_or_else(|poisoned| poisoned.into_inner())
1398 .has_deadlines();
1399 if !has_deadlines {
1400 self.scope_deadline_added.notified().await;
1401 continue;
1402 }
1403 tokio::time::sleep(Duration::from_secs(1)).await;
1404 if let Err(error) = self.sweep_expired_scopes() {
1405 warn!(%error, "scope expiry sweep failed");
1406 }
1407 }
1408 });
1409 }
1410
1411 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1412 let mut table = self
1413 .scopes
1414 .write()
1415 .unwrap_or_else(|poisoned| poisoned.into_inner());
1416 let swept = table.sweep_expired();
1417 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1418 drop(table);
1419 self.close_scope_drained_routes(drained);
1420 Ok(())
1421 }
1422
1423 fn publish_scope_changes(
1424 &self,
1425 changes: &[crate::scopes::ScopeTagChange],
1426 expired: &[crate::scopes::ScopeExpired],
1427 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1428 if changes.is_empty() {
1429 return Ok(Vec::new());
1430 }
1431 let drained = self
1432 .forwarding
1433 .publish_scope_changes(changes)
1434 .map_err(RouterError::Forwarding)?;
1435 let mut counts = HashMap::new();
1436 for route in &drained {
1437 *counts
1438 .entry((
1439 route.scope.owner.as_str(),
1440 route.scope.scope_ref.as_str(),
1441 route.scope.tag.scope_epoch,
1442 ))
1443 .or_insert(0usize) += 1;
1444 }
1445 for scope in expired {
1446 let routes_closed = counts
1447 .get(&(
1448 scope.owner.as_str(),
1449 scope.scope_ref.as_str(),
1450 scope.scope_epoch,
1451 ))
1452 .copied()
1453 .unwrap_or(0);
1454 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1455 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1456 routes_closed, "scope expired");
1457 }
1458 Ok(drained)
1459 }
1460
1461 fn runtime_capability_snapshot(
1462 &self,
1463 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1464 let runtime = self
1465 .supervisor
1466 .list()
1467 .into_iter()
1468 .map(|module| {
1469 let status = module.status().map_err(|err| {
1470 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1471 })?;
1472 Ok(RuntimeModule {
1473 module_id: status.module_id,
1474 state: status.state,
1475 enabled: status.enabled,
1476 })
1477 })
1478 .collect::<Result<Vec<_>, RouterError>>()?;
1479 let registrations = self.registry.list_registered_modules().map_err(|err| {
1485 RouterError::backend(
1486 0,
1487 0,
1488 format!("failed to list capability registrations: {err}"),
1489 )
1490 })?;
1491 let registrations = registrations
1492 .into_iter()
1493 .map(|registration| RegisteredModule {
1494 module_id: registration.manifest.module_id,
1495 module_version: registration.manifest.module_version,
1496 capabilities: registration.manifest.capabilities,
1497 })
1498 .collect();
1499 Ok((runtime, registrations))
1500 }
1501
1502 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1509 let cached_registration = RegisteredModule {
1510 module_id: registration.manifest.module_id.clone(),
1511 module_version: registration.manifest.module_version.clone(),
1512 capabilities: registration.manifest.capabilities.clone(),
1513 };
1514 let change = self.capability_evaluator.record_hello(&cached_registration);
1515 log_manifest_change(&cached_registration.module_id, &change);
1516 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1517 self.enforce_capability_denies();
1518 }
1519 self.refresh_capability_requirements();
1520 }
1521
1522 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1526 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1527 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1528 self.supervisor.set_swap_promotion_observer(observer);
1529 }
1530
1531 pub fn refresh_capability_requirements(&self) {
1532 match self.runtime_capability_snapshot() {
1533 Ok((runtime, registrations)) => {
1534 log_requirement_events(
1535 self.capability_evaluator
1536 .evaluate_now(&runtime, ®istrations),
1537 &self.daemon_shutdown,
1538 );
1539 }
1540 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1541 }
1542 }
1543
1544 fn enforce_capability_denies(&self) {
1548 let registrations = match self.registry.list_registered_modules() {
1549 Ok(snapshot) => snapshot,
1550 Err(err) => {
1551 warn!(error = %err, "failed to read registrations for capability deny census");
1552 return;
1553 }
1554 };
1555 let manifests = registrations
1556 .into_iter()
1557 .map(|registration| {
1558 (
1559 registration.manifest.module_id.clone(),
1560 registration.manifest,
1561 )
1562 })
1563 .collect::<BTreeMap<_, _>>();
1564 let census = match self.forwarding.route_census(None) {
1565 Ok(census) => census,
1566 Err(err) => {
1567 warn!(error = %err, "failed to read route census for capability deny enforcement");
1568 return;
1569 }
1570 };
1571
1572 for (target_module_id, routes) in census {
1573 let Some(target_manifest) = manifests.get(&target_module_id) else {
1574 continue;
1575 };
1576 let mut closed_routes = Vec::new();
1577 let mut goodbyes = Vec::new();
1578 for route in routes {
1579 let Principal::Reserved {
1580 module_id: opening_module_id,
1581 } = &route.principal
1582 else {
1583 continue;
1584 };
1585 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1586 continue;
1587 };
1588 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1589 continue;
1590 };
1591
1592 match self.forwarding.release_client_route(
1593 route.goodbye_target.connection_id,
1594 route.goodbye_target.channel,
1595 route.goodbye_target.epoch,
1596 ) {
1597 Ok(RouteRelease::Removed(module_goodbye)) => {
1598 warn!(
1599 opening_module_id,
1600 target_module_id,
1601 capability,
1602 "force-closing route because an attested capability deny edge now matches"
1603 );
1604 goodbyes.push(module_goodbye);
1605 goodbyes.push(route.goodbye_target.clone());
1606 closed_routes.push(route);
1607 }
1608 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1609 Err(err) => warn!(
1610 opening_module_id,
1611 target_module_id,
1612 capability,
1613 error = %err,
1614 "failed to force-close capability-denied route"
1615 ),
1616 }
1617 }
1618
1619 if closed_routes.is_empty() {
1620 continue;
1621 }
1622 send_route_control_pushes(
1623 &self.forwarding,
1624 closed_routes,
1625 ClientControlPush::RouteClosed {
1626 module_id: target_module_id,
1627 channels: Vec::new(),
1628 reason: RouteCloseReason::CapabilityDenied,
1629 drained: false,
1630 abandoned: 0,
1631 excluded_subscriptions: 0,
1632 terminal: Some(false),
1633 },
1634 );
1635 self.emit_route_goodbyes(goodbyes);
1638 }
1639 }
1640
1641 fn not_ready_reason(
1647 &self,
1648 registration: &crate::registry::ModuleRegistration,
1649 ) -> Option<NotReadyReason> {
1650 if !registration.ready {
1651 return Some(NotReadyReason {
1652 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1653 capability: None,
1654 });
1655 }
1656 self.first_unprovided_required_capability(registration)
1657 .map(|capability| NotReadyReason {
1658 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1659 capability: Some(capability),
1660 })
1661 }
1662
1663 fn first_unprovided_required_capability(
1681 &self,
1682 registration: &crate::registry::ModuleRegistration,
1683 ) -> Option<String> {
1684 let required = registration
1685 .manifest
1686 .capabilities
1687 .iter()
1688 .flat_map(|declarations| declarations.requires.iter())
1689 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1690 .map(|requirement| requirement.capability.as_str())
1691 .collect::<BTreeSet<_>>();
1692 if required.is_empty() {
1693 return None;
1694 }
1695 let module_id = registration.manifest.module_id.as_str();
1696 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1697 if required
1698 .iter()
1699 .any(|capability| verdict(capability).is_none())
1700 {
1701 self.refresh_capability_requirements();
1702 }
1703 required
1704 .into_iter()
1705 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1706 .map(str::to_string)
1707 }
1708
1709 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1710 self.capability_evaluator
1711 .statuses()
1712 .into_iter()
1713 .map(capability_requirement_status)
1714 .collect()
1715 }
1716
1717 fn deregister_connection(
1724 &self,
1725 connection_id: ConnectionId,
1726 reason: RegistrationEndReason,
1727 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1728 self.registry
1729 .deregister_connection_with_reason(connection_id, reason)
1730 }
1731
1732 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1733 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1734 return None;
1735 }
1736 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1737 parse_client_control_request(&frame.body)
1738 else {
1739 return None;
1740 };
1741 Some(target_module_id(&target).to_string())
1742 }
1743
1744 pub(crate) fn route_open_capacity_refusal(
1745 &self,
1746 ctx: &RouteCtx,
1747 frame: &Frame,
1748 target_module_id: &str,
1749 in_flight: usize,
1750 limit: usize,
1751 ) -> Result<Frame, RouterError> {
1752 let caller = self.route_open_caller_from_frame(frame);
1755 self.route_open_admission_refusal_frame(
1756 ctx,
1757 &caller,
1758 frame,
1759 target_module_id,
1760 "open_admission_full",
1761 (in_flight, limit),
1762 format!(
1763 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1764 ),
1765 )
1766 }
1767
1768 fn route_open_target_capacity_refusal(
1769 &self,
1770 ctx: &RouteCtx,
1771 caller: &RouteOpenCaller,
1772 frame: &Frame,
1773 target_module_id: &str,
1774 in_flight: usize,
1775 ) -> Result<Frame, RouterError> {
1776 self.route_open_admission_refusal_frame(
1777 ctx,
1778 caller,
1779 frame,
1780 target_module_id,
1781 "target_binds_full",
1782 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1783 format!(
1784 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1785 ),
1786 )
1787 }
1788
1789 #[allow(clippy::too_many_arguments)]
1797 fn route_open_admission_refusal_frame(
1798 &self,
1799 ctx: &RouteCtx,
1800 caller: &RouteOpenCaller,
1801 frame: &Frame,
1802 target_module_id: &str,
1803 reason: &'static str,
1804 (in_flight, limit): (usize, usize),
1805 message: impl Into<String>,
1806 ) -> Result<Frame, RouterError> {
1807 let code = error_codes::TARGET_UNAVAILABLE;
1808 self.counters.increment_route_open_refused(code);
1809 info!(
1810 target: "control",
1811 code,
1812 reason,
1813 module_id = ?target_module_id,
1814 connection_id = ctx.connection_id.get(),
1815 principal = %caller.principal,
1816 scope = %caller.scope,
1817 harness = %caller.harness,
1818 in_flight,
1819 limit,
1820 "route.open refused"
1821 );
1822 control_error_frame(frame, code, message.into())
1823 }
1824
1825 #[cfg(test)]
1831 pub fn handle_control(
1832 &self,
1833 connection_id: ConnectionId,
1834 frame: Frame,
1835 ) -> Result<Vec<Frame>, RouterError> {
1836 match frame.header.ty {
1837 FrameType::Ping => Ok(vec![pong(&frame)?]),
1838 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1839 FrameType::Goodbye => self.handle_goodbye(connection_id),
1840 ty => Ok(vec![control_error_frame(
1841 &frame,
1842 "unsupported_control_frame",
1843 format!("unsupported channel-0 frame {ty:?}"),
1844 )?]),
1845 }
1846 }
1847
1848 pub async fn handle_control_frame(
1849 &self,
1850 ctx: &RouteCtx,
1851 frame: Frame,
1852 ) -> Result<Vec<Frame>, RouterError> {
1853 self.handle_control_frame_timed(ctx, frame, None).await
1854 }
1855
1856 #[cfg(any(test, feature = "test-support"))]
1857 pub fn with_test_operator_app_policy(
1858 mut self,
1859 policy: crate::operator_apps::TestOperatorAppPolicy,
1860 ) -> Self {
1861 self.operator_app_policy = policy.into();
1862 self
1863 }
1864
1865 #[cfg(all(windows, any(test, feature = "test-support")))]
1869 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
1870 self.windows_verify = Some(verify);
1871 self
1872 }
1873
1874 #[cfg(all(windows, any(test, feature = "test-support")))]
1876 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
1877 self.windows_flash = Some(flash);
1878 self
1879 }
1880
1881 #[cfg(all(windows, any(test, feature = "test-support")))]
1885 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
1886 self.windows_track = Some(track);
1887 self
1888 }
1889
1890 pub(crate) fn record_captured_peer(
1891 &self,
1892 connection: ConnectionId,
1893 peer: subc_os::local_peer::CapturedPeer,
1894 ) {
1895 self.record_observed_peer(connection, peer.observation().clone());
1896 #[cfg(windows)]
1897 if let Some(stamp) = self.stamp_windows_launch(&peer) {
1898 self.operator_apps
1899 .lock()
1900 .unwrap_or_else(|p| p.into_inner())
1901 .insert(connection, stamp);
1902 }
1903 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1904 self.operator_apps
1905 .lock()
1906 .unwrap_or_else(|p| p.into_inner())
1907 .insert(connection, stamp);
1908 }
1909 }
1910
1911 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
1912 self.local_connections
1913 .lock()
1914 .unwrap_or_else(|p| p.into_inner())
1915 .insert(connection);
1916 }
1917
1918 fn connection_operator_app(
1921 &self,
1922 stamp: Option<&crate::operator_apps::OperatorAppStamp>,
1923 ) -> Option<subc_protocol::peer::OperatorApp> {
1924 stamp
1925 .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
1926 .map(|stamp| stamp.app.clone())
1927 }
1928
1929 fn handle_connection_describe(
1930 &self,
1931 ctx: &RouteCtx,
1932 frame: Frame,
1933 ) -> Result<Vec<Frame>, RouterError> {
1934 if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
1935 return Ok(vec![control_error_frame(
1936 &frame,
1937 "invalid_control_body",
1938 "malformed connection.describe body",
1939 )?]);
1940 }
1941 let local = self
1942 .local_connections
1943 .lock()
1944 .unwrap_or_else(|p| p.into_inner())
1945 .contains(&ctx.connection_id);
1946 let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
1947 let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
1948 Ok(vec![control_response_body_frame(
1949 &frame,
1950 &subc_control::ConnectionDescribeReply::new(local, operator_app),
1951 ops::CONNECTION_DESCRIBE,
1952 )?])
1953 }
1954
1955 pub(crate) fn record_observed_peer(
1956 &self,
1957 connection: ConnectionId,
1958 peer: subc_os::local_peer::ObservedPeer,
1959 ) {
1960 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1961 let peer = match peer {
1962 OsPeer::Unobserved => ObservedPeer::unobserved(),
1963 OsPeer::Observed(peer) => {
1964 let observed = match peer.user {
1965 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1966 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1967 };
1968 observed
1969 .with_exe(
1970 peer.exe
1971 .and_then(|path| path.into_os_string().into_string().ok()),
1972 )
1973 .with_signing_id(peer.signing_id)
1974 }
1975 };
1976 self.observed_peers
1977 .lock()
1978 .unwrap_or_else(|p| p.into_inner())
1979 .insert(connection, peer);
1980 }
1981
1982 pub(crate) fn record_caller_endpoints(
1983 &self,
1984 connection: ConnectionId,
1985 endpoints: Option<(SocketAddr, SocketAddr)>,
1986 ) {
1987 let mut callers = self
1988 .caller_endpoints
1989 .lock()
1990 .unwrap_or_else(|p| p.into_inner());
1991 callers.remove(&connection);
1992 if let Some(endpoints) = endpoints {
1993 callers.insert(connection, endpoints);
1994 }
1995 }
1996
1997 #[cfg(test)]
1998 pub(crate) fn with_caller_lookup_for_test(
1999 mut self,
2000 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
2001 ) -> Self {
2002 self.caller_lookup = Some(Arc::new(lookup));
2003 self
2004 }
2005
2006 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
2007 let endpoints = self
2008 .caller_endpoints
2009 .lock()
2010 .unwrap_or_else(|p| p.into_inner())
2011 .get(&connection)
2012 .copied();
2013 #[cfg(test)]
2014 let lookup = self.caller_lookup.clone();
2015 tokio::task::spawn_blocking(move || {
2016 #[cfg(test)]
2017 if let Some(lookup) = lookup {
2018 return lookup(endpoints);
2019 }
2020 endpoints
2021 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2022 .unwrap_or_default()
2023 })
2024 .await
2025 .unwrap_or_default()
2026 }
2027
2028 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2029 frame.header.channel == 0
2030 && frame.header.ty == FrameType::Request
2031 && self
2032 .forwarding
2033 .module_endpoint_for_connection(ctx.connection_id)
2034 .is_ok_and(|endpoint| endpoint.is_none())
2035 && parse_client_control_request(&frame.body)
2036 .ok()
2037 .is_some_and(|request| privileged_operation(&request).is_some())
2038 }
2039
2040 pub(crate) async fn handle_control_frame_timed(
2041 &self,
2042 ctx: &RouteCtx,
2043 frame: Frame,
2044 dispatch_started_at: Option<StdInstant>,
2045 ) -> Result<Vec<Frame>, RouterError> {
2046 match frame.header.ty {
2047 FrameType::Ping => Ok(vec![pong(&frame)?]),
2048 FrameType::Hello => {
2049 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
2050 }
2051 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2052 FrameType::Cancel => {
2053 if self
2056 .agent_declarations
2057 .cancel(ctx.connection_id, frame.header.corr)
2058 || self
2059 .forwarding
2060 .operator_confirms()
2061 .cancel(ctx.connection_id, frame.header.corr)
2062 || self
2063 .supervisor
2064 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2065 || self.cancel_activity_subscription(ctx, &frame)?
2066 {
2067 Ok(Vec::new())
2068 } else {
2069 Ok(vec![control_error_frame(
2070 &frame,
2071 "unknown_subscription",
2072 "no subscription has this correlation id",
2073 )?])
2074 }
2075 }
2076 FrameType::Request => {
2077 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2080 if op
2081 .as_ref()
2082 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2083 {
2084 return self.handle_activity_record(ctx, frame).await;
2085 }
2086 if op.as_ref().is_some_and(|probe| {
2087 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2088 }) {
2089 return self.handle_activity_read(ctx, frame).await;
2090 }
2091 if op
2092 .as_ref()
2093 .is_some_and(|probe| probe.op == "operator.confirm")
2094 {
2095 return self.handle_operator_confirm(ctx, frame);
2096 }
2097 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2098 return self
2099 .observe_control_wait(
2100 ctx,
2101 None,
2102 ops::APP_OPEN,
2103 self.handle_app_open(ctx, frame),
2104 )
2105 .await;
2106 }
2107 if op
2108 .as_ref()
2109 .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2110 {
2111 return self.handle_connection_describe(ctx, frame);
2112 }
2113 if op
2114 .as_ref()
2115 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2116 {
2117 return self.handle_scope_served_by(ctx.connection_id, frame);
2118 }
2119 if self
2120 .forwarding
2121 .module_endpoint_for_connection(ctx.connection_id)
2122 .map_err(RouterError::Forwarding)?
2123 .is_some()
2124 {
2125 if !is_known_module_request_op(&frame.body) {
2126 return Ok(vec![control_error_frame(
2127 &frame,
2128 "unsupported_control_frame",
2129 "module-originated channel-0 REQUEST is not supported",
2130 )?]);
2131 }
2132 let request = match parse_module_control_request_from_module(&frame.body) {
2133 Ok(request) => request,
2134 Err((err, ControlRequestBodyError::UnknownOp)) => {
2135 return Ok(vec![control_error_frame(
2136 &frame,
2137 "unsupported_control_frame",
2138 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2139 )?])
2140 }
2141 Err((err, ControlRequestBodyError::InvalidBody)) => {
2142 return Ok(vec![control_error_frame(
2143 &frame,
2144 "invalid_control_body",
2145 format!("malformed module control body: {err}"),
2146 )?])
2147 }
2148 };
2149 let op = module_control_request_op(&request);
2150 let corr = frame.header.corr;
2151 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2152 if let ModuleControlRequestFromModule::AgentCommands {
2153 started,
2154 finished,
2155 known,
2156 } = request
2157 {
2158 return self
2159 .agent_declarations
2160 .enqueue(
2161 self,
2162 ctx,
2163 frame,
2164 crate::resource_totals::Declaration {
2165 started,
2166 finished,
2167 known,
2168 },
2169 )
2170 .await;
2171 }
2172 let result =
2173 self.handle_module_control_request(ctx.connection_id, frame, request);
2174 log_slow_control_dispatch(
2175 dispatch_started_at,
2176 op,
2177 ctx.connection_id,
2178 corr,
2179 None,
2180 );
2181 return result;
2182 }
2183
2184 if is_known_module_request_op(&frame.body) {
2185 let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2186 .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2187 let code = if is_agents {
2188 "not_supervised"
2189 } else if op
2190 .as_ref()
2191 .is_some_and(|probe| probe.op == subc_protocol::scope::ROUTE_AUTHORIZE_OP)
2192 {
2193 error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED
2194 } else {
2195 "not_registered"
2196 };
2197 return Ok(vec![control_error_frame(
2198 &frame,
2199 code,
2200 "module-originated control requests require the module's own registered connection",
2201 )?]);
2202 }
2203
2204 let request = match parse_client_control_request(&frame.body) {
2205 Ok(request) => request,
2206 Err((err, ControlRequestBodyError::UnknownOp)) => {
2207 return Ok(vec![control_error_frame(
2208 &frame,
2209 "unknown_control_op",
2210 format!("unknown client control op: {err}"),
2211 )?])
2212 }
2213 Err((err, ControlRequestBodyError::InvalidBody)) => {
2214 return Ok(vec![control_error_frame(
2215 &frame,
2216 "invalid_control_body",
2217 format!("malformed client control body: {err}"),
2218 )?])
2219 }
2220 };
2221 let op = client_control_request_op(&request);
2222 let corr = frame.header.corr;
2223 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2224 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2225 #[cfg(test)]
2226 if let Some(delay) = self.control_dispatch_delay {
2227 tokio::time::sleep(delay).await;
2228 }
2229 let result = self
2230 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2231 .await;
2232 if let Some(timings) = route_open_timings.as_mut() {
2233 timings.finish();
2234 }
2235 log_slow_control_dispatch(
2236 dispatch_started_at,
2237 op,
2238 ctx.connection_id,
2239 corr,
2240 route_open_timings.as_ref(),
2241 );
2242 result
2243 }
2244 FrameType::Push => {
2245 let Some(endpoint) = self
2246 .forwarding
2247 .module_endpoint_for_connection(ctx.connection_id)
2248 .map_err(RouterError::Forwarding)?
2249 else {
2250 return Ok(vec![control_error_frame(
2251 &frame,
2252 "unsupported_control_frame",
2253 "client-originated channel-0 PUSH is not supported",
2254 )?]);
2255 };
2256 self.handle_status_update(endpoint, frame)
2257 }
2258 FrameType::Response | FrameType::Error
2259 if self
2260 .forwarding
2261 .module_endpoint_for_connection(ctx.connection_id)
2262 .map_err(RouterError::Forwarding)?
2263 .is_some() =>
2264 {
2265 self.handle_module_relay_response(ctx.connection_id, frame)
2266 }
2267 ty => Ok(vec![control_error_frame(
2268 &frame,
2269 "unsupported_control_frame",
2270 format!("unsupported channel-0 frame {ty:?}"),
2271 )?]),
2272 }
2273 }
2274
2275 pub fn cleanup_connection(
2276 &self,
2277 connection_id: ConnectionId,
2278 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2279 self.local_connections
2280 .lock()
2281 .unwrap_or_else(|p| p.into_inner())
2282 .remove(&connection_id);
2283 self.caller_endpoints
2284 .lock()
2285 .unwrap_or_else(|p| p.into_inner())
2286 .remove(&connection_id);
2287 self.build_class_warned
2288 .lock()
2289 .unwrap_or_else(|p| p.into_inner())
2290 .remove(&connection_id);
2291 self.observed_peers
2292 .lock()
2293 .unwrap_or_else(|p| p.into_inner())
2294 .remove(&connection_id);
2295 self.operator_apps
2296 .lock()
2297 .unwrap_or_else(|p| p.into_inner())
2298 .remove(&connection_id);
2299 self.forwarding
2300 .operator_confirms()
2301 .connection_closed(connection_id);
2302 self.cleanup_connection_with_end_reason(
2303 connection_id,
2304 RegistrationEndReason::ConnectionClosed,
2305 )
2306 }
2307
2308 fn cleanup_connection_with_end_reason(
2309 &self,
2310 connection_id: ConnectionId,
2311 requested_reason: RegistrationEndReason,
2312 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2313 self.agent_declarations.close(connection_id);
2314 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2315 self.registry
2316 .get_module_by_connection(connection_id)?
2317 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2318 .and_then(|module| module.registration_end_reason().ok().flatten())
2319 .unwrap_or(requested_reason)
2320 } else {
2321 requested_reason
2322 };
2323 let crash_closed = self
2324 .registry
2325 .get_module_by_connection(connection_id)?
2326 .and_then(|registration| {
2327 self.forwarding
2328 .module_endpoint_for_connection(connection_id)
2329 .ok()
2330 .flatten()
2331 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2332 .map(|routes| (registration.manifest.module_id, routes))
2333 });
2334 let crash_closed = crash_closed.map(|(module_id, routes)| {
2335 let terminal = match self.supervisor.get(&module_id) {
2336 None => false,
2337 Some(module) => match module.will_recover_after_connection_loss() {
2338 Ok(will_recover) => !will_recover,
2339 Err(err) => {
2340 warn!(
2341 %module_id,
2342 error = %err,
2343 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2344 );
2345 false
2346 }
2347 },
2348 };
2349 let (reason, drained) = match self
2353 .forwarding
2354 .daemon_drain_result_for_connection(connection_id)
2355 {
2356 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2357 Ok(None) => (RouteCloseReason::Crash, false),
2358 Err(err) => {
2359 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2360 (RouteCloseReason::Crash, false)
2361 }
2362 };
2363 (module_id, routes, reason, drained, terminal)
2364 });
2365 let registrations = self.deregister_connection(connection_id, end_reason);
2366 let cleanup = if crash_closed.is_some() {
2367 self.forwarding.cleanup_connection_counted(connection_id)
2368 } else {
2369 self.forwarding
2375 .cleanup_connection(connection_id)
2376 .map(|released| crate::forwarding::ConnectionCleanup {
2377 released,
2378 abandoned_relays: 0,
2379 })
2380 };
2381 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2386 let abandoned = cleanup
2387 .as_ref()
2388 .map_or(0, |cleanup| cleanup.abandoned_relays);
2389 send_route_control_pushes(
2390 &self.forwarding,
2391 routes,
2392 ClientControlPush::RouteClosed {
2393 module_id,
2394 channels: Vec::new(),
2395 reason,
2396 drained,
2397 abandoned,
2398 excluded_subscriptions: 0,
2399 terminal: Some(terminal),
2400 },
2401 );
2402 }
2403 if let Ok(cleanup) = cleanup {
2404 self.emit_route_goodbyes(cleanup.released);
2405 }
2406 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2410 crate::supervise::notify_registration_release();
2411 self.capability_evaluator.wake_deadline_loop();
2412 self.refresh_capability_requirements();
2413 }
2414 self.supervisor.remove_spawn_subscribers(connection_id);
2415 self.activity_streams
2416 .lock()
2417 .unwrap_or_else(|p| p.into_inner())
2418 .retain(|(id, _), task| {
2419 if *id == connection_id {
2420 task.abort();
2421 false
2422 } else {
2423 true
2424 }
2425 });
2426 self.hello_launch_nonces
2429 .lock()
2430 .unwrap_or_else(|poisoned| poisoned.into_inner())
2431 .forget(connection_id);
2432 self.scopes
2433 .write()
2434 .unwrap_or_else(|poisoned| poisoned.into_inner())
2435 .release_connection(connection_id);
2436 registrations
2437 }
2438
2439 pub(crate) fn handle_route_goodbye(
2440 &self,
2441 connection_id: ConnectionId,
2442 route_channel: u16,
2443 route_epoch: u32,
2444 ) -> Result<bool, RouterError> {
2445 debug!(
2446 connection_id = connection_id.get(),
2447 route_channel, route_epoch, "handling route GOODBYE"
2448 );
2449 let RouteRelease::Removed(released_route) = self
2450 .forwarding
2451 .release_client_route(connection_id, route_channel, route_epoch)
2452 .map_err(RouterError::Forwarding)?
2453 else {
2454 return Ok(false);
2455 };
2456 self.emit_route_goodbyes(vec![released_route]);
2457 Ok(true)
2458 }
2459
2460 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2461 for released in released_routes {
2462 let frame = match Frame::build_with_version(
2463 released.negotiated_ver,
2464 FrameType::Goodbye,
2465 control_flags(),
2466 released.channel,
2467 released.epoch,
2468 0,
2469 Vec::new(),
2470 ) {
2471 Ok(frame) => frame,
2472 Err(err) => {
2473 warn!(
2474 route_channel = released.channel,
2475 error = %err,
2476 "failed to build route GOODBYE frame"
2477 );
2478 continue;
2479 }
2480 };
2481 if !released.close_on_delivery_failure() {
2482 crate::forwarding::send_module_route_goodbye(
2483 &self.counters,
2484 &released.sink,
2485 frame,
2486 released.module_id.as_deref(),
2487 "client route released",
2488 );
2489 continue;
2490 }
2491 if let Err(err) = released.sink.try_send(frame) {
2492 warn!(
2493 target_connection_id = released.connection_id.get(),
2494 route_channel = released.channel,
2495 error = %err,
2496 "route GOODBYE was not delivered to client; closing target connection"
2497 );
2498 if self
2499 .forwarding
2500 .escalate_client_delivery_failure(
2501 released.connection_id,
2502 released.channel,
2503 released.epoch,
2504 CloseReason::new(
2505 "route_goodbye_delivery_failed",
2506 format!(
2507 "failed to enqueue route GOODBYE for channel {}: {err}",
2508 released.channel
2509 ),
2510 ),
2511 crate::forwarding::UndeliveredFrame {
2512 module_id: released.module_id.as_deref(),
2513 sink: &released.sink,
2514 },
2515 )
2516 .unwrap_or(false)
2517 {
2518 self.counters.increment_goodbye_relay_client_failed();
2519 }
2520 }
2521 }
2522 }
2523
2524 fn send_abandoned_route_bind_goodbye(
2536 &self,
2537 module_sink: &crate::FrameSink,
2538 negotiated_ver: u8,
2539 module_channel: u16,
2540 module_epoch: u32,
2541 ) {
2542 let frame = match Frame::build_with_version(
2543 negotiated_ver,
2544 FrameType::Goodbye,
2545 control_flags(),
2546 module_channel,
2547 module_epoch,
2548 0,
2549 Vec::new(),
2550 ) {
2551 Ok(frame) => frame,
2552 Err(err) => {
2553 warn!(
2554 route_channel = module_channel,
2555 error = %err,
2556 "failed to build GOODBYE for abandoned route.bind"
2557 );
2558 return;
2559 }
2560 };
2561 crate::forwarding::send_module_route_goodbye(
2562 &self.counters,
2563 module_sink,
2564 frame,
2565 None,
2566 "abandoned route.bind",
2567 );
2568 }
2569
2570 fn handle_hello(
2571 &self,
2572 connection_id: ConnectionId,
2573 sink: Option<crate::FrameSink>,
2574 frame: Frame,
2575 ) -> Result<Vec<Frame>, RouterError> {
2576 debug!(
2577 connection_id = connection_id.get(),
2578 corr = frame.header.corr,
2579 "handling HELLO"
2580 );
2581 if self
2585 .registry
2586 .get_module_by_connection(connection_id)
2587 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2588 .is_some()
2589 {
2590 return Ok(vec![control_error_frame(
2591 &frame,
2592 "invalid_hello",
2593 "connection is already registered as a module",
2594 )?]);
2595 }
2596 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2597 Ok(value) => value,
2598 Err(err) => {
2599 return Ok(vec![control_error_frame(
2600 &frame,
2601 "invalid_hello",
2602 format!("malformed HELLO body: {err}"),
2603 )?])
2604 }
2605 };
2606 if let Err(err) = validate_hello_display_name(&hello_value) {
2607 return Ok(vec![control_error_frame(
2608 &frame,
2609 "invalid_manifest",
2610 err.to_string(),
2611 )?]);
2612 }
2613 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2614 return Ok(vec![control_error_frame(
2615 &frame,
2616 "invalid_capability_grammar",
2617 err.to_string(),
2618 )?]);
2619 }
2620 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2621 return Ok(vec![control_error_frame(
2622 &frame,
2623 "invalid_manifest",
2624 err.to_string(),
2625 )?]);
2626 }
2627 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2628 return Ok(vec![control_error_frame(
2629 &frame,
2630 "invalid_event_declaration",
2631 err.to_string(),
2632 )?]);
2633 }
2634 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2635 return Ok(vec![control_error_frame(
2636 &frame,
2637 "invalid_manifest",
2638 err.to_string(),
2639 )?]);
2640 }
2641 if let Some(provenance) = hello_value
2642 .get("manifest")
2643 .and_then(|manifest| manifest.get("provenance"))
2644 {
2645 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2646 return Ok(vec![control_error_frame(
2647 &frame,
2648 "invalid_manifest",
2649 format!("malformed manifest provenance: {err}"),
2650 )?]);
2651 }
2652 }
2653 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2654 let module_id = hello_value["manifest"]["module_id"]
2655 .as_str()
2656 .unwrap_or("<unknown>");
2657 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2658 }
2659 if let Err(reason) = validate_hello_operation_deferral(&hello_value) {
2660 let module_id = hello_value
2661 .get("manifest")
2662 .and_then(|m| m.get("module_id"))
2663 .and_then(serde_json::Value::as_str)
2664 .unwrap_or("unknown");
2665 warn!(module_id, %reason, "dropping operation_deferral from HELLO");
2666 }
2667 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2668 let module_id = hello_value["manifest"]["module_id"]
2669 .as_str()
2670 .unwrap_or("<unknown>");
2671 warn!(module_id, %reason, "dropping flow_operations from HELLO");
2672 }
2673 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2674 let module_id = hello_value["manifest"]["module_id"]
2675 .as_str()
2676 .unwrap_or("<unknown>");
2677 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2678 }
2679 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2680 Ok(hello) => hello,
2681 Err(err) => {
2682 return Ok(vec![control_error_frame(
2683 &frame,
2684 "invalid_hello",
2685 format!("malformed HELLO body: {err}"),
2686 )?])
2687 }
2688 };
2689
2690 if hello.protocol_ver != hello.manifest.protocol_ver {
2691 return Ok(vec![control_error_frame(
2692 &frame,
2693 "invalid_manifest",
2694 format!(
2695 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2696 hello.protocol_ver, hello.manifest.protocol_ver
2697 ),
2698 )?]);
2699 }
2700
2701 if hello.manifest.module_id.trim().is_empty() {
2702 return Ok(vec![control_error_frame(
2703 &frame,
2704 "invalid_manifest",
2705 "manifest module_id must not be empty",
2706 )?]);
2707 }
2708
2709 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2710 Ok(negotiated_ver) => negotiated_ver,
2711 Err(message) => {
2712 return Ok(vec![control_error_frame(
2713 &frame,
2714 "version_unsupported",
2715 message,
2716 )?])
2717 }
2718 };
2719
2720 let swap_admission = self
2728 .supervisor
2729 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2730 if swap_admission == SwapHelloAdmission::Refused {
2731 warn!(
2732 module_id = %hello.manifest.module_id,
2733 connection_id = connection_id.get(),
2734 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2735 );
2736 return Ok(vec![control_error_frame(
2737 &frame,
2738 "swap_token_invalid",
2739 format!(
2740 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2741 hello.manifest.module_id
2742 ),
2743 )?]);
2744 }
2745 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2746
2747 let verification = if swap_candidate {
2752 ReservedHelloVerification {
2753 nonce_verified: true,
2754 prefix_owner: None,
2755 }
2756 } else {
2757 match self.supervisor.reserved_hello_verification(
2758 &hello.manifest.module_id,
2759 hello.launch_nonce.as_deref(),
2760 ) {
2761 Ok(verified) => verified,
2762 Err(rejection) => {
2763 let message = match rejection {
2764 ReservedHelloRejection::Exact { module_id } => format!(
2765 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2766 ),
2767 ReservedHelloRejection::Prefix {
2768 prefix,
2769 owner_module_id,
2770 } => format!(
2771 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2772 hello.manifest.module_id
2773 ),
2774 };
2775 return Ok(vec![control_error_frame(
2776 &frame,
2777 "reserved_module",
2778 message,
2779 )?]);
2780 }
2781 }
2782 };
2783
2784 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2785 &hello.manifest.module_id,
2786 hello.manifest.capabilities.as_ref(),
2787 );
2788 if let Some(refusal) = reserved_capability_refusals.first() {
2789 let capability = refusal.capability.clone();
2790 let bound_module = refusal.claimants[0].clone();
2791 log_duplicate_claim_events(reserved_capability_refusals);
2792 return Ok(vec![control_error_frame(
2793 &frame,
2794 "reserved_capability",
2795 format!(
2796 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2797 capability, bound_module, hello.manifest.module_id
2798 ),
2799 )?]);
2800 }
2801
2802 if self
2805 .forwarding
2806 .connection_has_client_routes(connection_id)
2807 .map_err(RouterError::Forwarding)?
2808 {
2809 return Ok(vec![control_error_frame(
2810 &frame,
2811 "invalid_hello",
2812 "connection has open client routes and cannot also register as a module",
2813 )?]);
2814 }
2815
2816 self.hello_launch_nonces
2821 .lock()
2822 .unwrap_or_else(|poisoned| poisoned.into_inner())
2823 .record(connection_id, hello.launch_nonce.as_deref());
2824 let control_ops = effective_module_control_ops(hello.control_ops);
2825 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2828 if swap_candidate {
2829 return self.register_swap_candidate(
2830 connection_id,
2831 sink,
2832 &frame,
2833 hello.manifest,
2834 negotiated_ver,
2835 control_ops,
2836 hello_ack,
2837 );
2838 }
2839 let registration = match self
2846 .registry
2847 .register_unpublished_verified_with_control_ops(
2848 hello.manifest,
2849 negotiated_ver,
2850 connection_id,
2851 control_ops,
2852 verification.nonce_verified,
2853 ) {
2854 Ok(registration) => registration,
2855 Err(RegistryError::DuplicateModuleId { module_id }) => {
2856 return Ok(vec![control_error_frame(
2857 &frame,
2858 "duplicate_module_id",
2859 format!(
2860 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2861 ),
2862 )?])
2863 }
2864 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2865 return Ok(vec![control_error_frame(
2866 &frame,
2867 "invalid_module_id",
2868 err.to_string(),
2869 )?])
2870 }
2871 Err(err) => {
2872 return Ok(vec![control_error_frame(
2873 &frame,
2874 "registry_error",
2875 err.to_string(),
2876 )?])
2877 }
2878 };
2879 #[cfg(test)]
2880 self.hold_between_registry_and_forwarding();
2881
2882 let reply = if let Some(sink) = sink {
2883 let concurrency = manifest_concurrency(®istration.manifest);
2899 if let Err(err) = self.forwarding.register_module_connection_acked(
2900 connection_id,
2901 registration.manifest.module_id.clone(),
2902 negotiated_ver,
2903 concurrency,
2904 sink,
2905 crate::forwarding::ModuleHelloAck {
2906 frame: hello_ack,
2907 verified_prefix_owner: verification.prefix_owner,
2908 },
2909 ) {
2910 if matches!(
2915 self.deregister_connection(
2916 connection_id,
2917 RegistrationEndReason::RegistrationFailed,
2918 ),
2919 Ok(r) if !r.is_empty()
2920 ) {
2921 crate::supervise::notify_registration_release();
2922 }
2923 return Ok(vec![control_error_frame(
2924 &frame,
2925 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2926 "invalid_hello"
2927 } else {
2928 forwarding_error_code(&err)
2929 },
2930 err.to_string(),
2931 )?]);
2932 }
2933 Vec::new()
2934 } else {
2935 vec![hello_ack]
2938 };
2939
2940 if !self
2945 .registry
2946 .publish_registration(connection_id)
2947 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2948 {
2949 debug!(
2950 module_id = %registration.manifest.module_id,
2951 connection_id = connection_id.get(),
2952 "registration was removed before it could be published"
2953 );
2954 }
2955
2956 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2965 info!(
2966 module_id = %registration.manifest.module_id,
2967 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2968 );
2969 }
2970
2971 self.apply_registration_capabilities(®istration);
2972
2973 self.supervisor
2974 .record_activity_registration(®istration.manifest.module_id);
2975
2976 info!(
2977 module_id = %registration.manifest.module_id,
2978 module_version = %registration.manifest.module_version,
2979 negotiated_ver,
2980 routable_provider = manifest_provides_routable_role(®istration.manifest),
2981 connection_id = connection_id.get(),
2982 "module registered"
2983 );
2984
2985 Ok(reply)
2986 }
2987
2988 #[allow(clippy::too_many_arguments)]
3005 fn register_swap_candidate(
3006 &self,
3007 connection_id: ConnectionId,
3008 sink: Option<crate::FrameSink>,
3009 frame: &Frame,
3010 manifest: ModuleManifest,
3011 negotiated_ver: u8,
3012 control_ops: Vec<String>,
3013 hello_ack: Frame,
3014 ) -> Result<Vec<Frame>, RouterError> {
3015 let module_id = manifest.module_id.clone();
3016 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
3017 manifest,
3018 negotiated_ver,
3019 connection_id,
3020 control_ops,
3021 ) {
3022 Ok(registration) => registration,
3023 Err(RegistryError::DuplicateModuleId { module_id }) => {
3024 return Ok(vec![control_error_frame(
3025 frame,
3026 "duplicate_module_id",
3027 format!(
3028 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
3029 ),
3030 )?])
3031 }
3032 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
3033 return Ok(vec![control_error_frame(
3034 frame,
3035 "invalid_module_id",
3036 err.to_string(),
3037 )?])
3038 }
3039 Err(err) => {
3040 return Ok(vec![control_error_frame(
3041 frame,
3042 "registry_error",
3043 err.to_string(),
3044 )?])
3045 }
3046 };
3047 #[cfg(test)]
3048 self.hold_between_registry_and_forwarding();
3049 let reply = if let Some(sink) = sink {
3050 let concurrency = manifest_concurrency(®istration.manifest);
3054 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
3055 connection_id,
3056 module_id.clone(),
3057 negotiated_ver,
3058 concurrency,
3059 sink,
3060 hello_ack,
3061 ) {
3062 if matches!(
3063 self.deregister_connection(
3064 connection_id,
3065 RegistrationEndReason::RegistrationFailed,
3066 ),
3067 Ok(r) if !r.is_empty()
3068 ) {
3069 crate::supervise::notify_registration_release();
3070 }
3071 return Ok(vec![control_error_frame(
3072 frame,
3073 forwarding_error_code(&err),
3074 err.to_string(),
3075 )?]);
3076 }
3077 Vec::new()
3078 } else {
3079 vec![hello_ack]
3080 };
3081 if !self
3082 .registry
3083 .publish_registration(connection_id)
3084 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3085 {
3086 debug!(
3087 module_id = %module_id,
3088 connection_id = connection_id.get(),
3089 "swap candidate registration was removed before it could be published"
3090 );
3091 }
3092 self.supervisor.mark_swap_candidate_admitted(&module_id);
3093 info!(
3094 module_id = %module_id,
3095 module_version = %registration.manifest.module_version,
3096 negotiated_ver,
3097 ready = registration.ready,
3098 connection_id = connection_id.get(),
3099 "swap candidate registered; not routable until cutover"
3100 );
3101 Ok(reply)
3102 }
3103
3104 fn build_hello_ack(
3105 &self,
3106 frame: &Frame,
3107 negotiated_ver: u8,
3108 module_id: &str,
3109 ) -> Result<Frame, RouterError> {
3110 let ack = ModuleHelloAckBody {
3111 negotiated_ver,
3112 subc_ops: module_subc_ops(),
3113 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
3114 storage: self
3115 .storage_config
3116 .as_ref()
3117 .map(|cfg| cfg.descriptor_for(module_id)),
3118 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3119 };
3120 let body = serde_json::to_vec(&ack).map_err(|err| {
3121 RouterError::backend(
3122 0,
3123 frame.header.corr,
3124 format!("failed to encode HELLO_ACK: {err}"),
3125 )
3126 })?;
3127
3128 Frame::build_with_version(
3129 negotiated_ver,
3130 FrameType::HelloAck,
3131 control_flags(),
3132 0,
3133 0,
3134 frame.header.corr,
3135 body,
3136 )
3137 .map_err(RouterError::FrameBuild)
3138 }
3139
3140 async fn handle_client_control_request(
3141 &self,
3142 ctx: &RouteCtx,
3143 frame: Frame,
3144 request: ClientControlRequest,
3145 route_open_timings: &mut Option<RouteOpenTimings>,
3146 ) -> Result<Vec<Frame>, RouterError> {
3147 if let Some(operation) = privileged_operation(&request) {
3148 let caller = self.resolve_caller(ctx.connection_id).await;
3149 let op = client_control_request_op(&request);
3150 let text = prompt_text(operation, &caller);
3151 match self.privileged_operator_prompt {
3152 PrivilegedOperatorPrompt::Log => {
3153 info!(target: "subc_daemon::privileged_prompt", operation = op,
3154 target = ?operation, caller = %caller.description(), summary = %text,
3155 "privileged request would prompt the operator");
3156 }
3157 PrivilegedOperatorPrompt::Enforce => {
3158 let key =
3159 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
3160 let stamp = self
3161 .operator_apps
3162 .lock()
3163 .unwrap_or_else(|p| p.into_inner())
3164 .get(&ctx.connection_id)
3165 .cloned();
3166 let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
3167 &self.operator_app_policy,
3168 &self.launch_table,
3169 stamp.as_ref(),
3170 )
3171 .then(|| self.shortcut_context(stamp, None, None));
3172 let outcome = self
3173 .forwarding
3174 .operator_confirms()
3175 .approve_privileged_with_shortcut(
3176 ctx,
3177 &frame,
3178 op,
3179 text,
3180 caller.description(),
3181 key,
3182 shortcut,
3183 )
3184 .await;
3185 if outcome.code != "confirmed" {
3186 return Ok(vec![outcome.frame(&frame)]);
3187 }
3188 }
3189 }
3190 }
3191 match request {
3192 ClientControlRequest::DaemonShutdownIntent { intent } => {
3193 if intent.as_str() != Some("restart") {
3194 return Ok(vec![control_error_frame(
3195 &frame,
3196 "invalid_params",
3197 "intent must be 'restart'",
3198 )?]);
3199 }
3200 let response = ClientControlResponse::DaemonShutdownIntent {
3201 intent: "restart".to_owned(),
3202 expires_at_ms: self.supervisor.announce_daemon_restart(),
3203 };
3204 Ok(vec![control_response_body_frame(
3205 &frame,
3206 &response,
3207 "ClientControlResponse::DaemonShutdownIntent",
3208 )?])
3209 }
3210 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
3211 ClientControlRequest::CatalogList { module_id } => {
3212 self.handle_catalog_list(frame, module_id)
3213 }
3214 ClientControlRequest::RouteOpen {
3215 target,
3216 identity,
3217 consumer_identity,
3218 consumer_capabilities,
3219 role_versions,
3220 busy_refusals,
3221 admission_facts,
3222 scope,
3223 ..
3224 } => {
3225 self.handle_route_open(
3226 ctx,
3227 frame,
3228 RouteOpenRequest {
3229 target,
3230 identity,
3231 consumer_identity,
3232 consumer_capabilities,
3233 role_versions,
3234 busy_refusals,
3235 admission_facts,
3236 scope,
3237 },
3238 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
3239 )
3240 .await
3241 }
3242 ClientControlRequest::RoutePoll {
3243 route_channel,
3244 route_epoch,
3245 kind,
3246 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
3247 ClientControlRequest::SupervisorList {} => {
3248 let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
3249 self.agent_declarations.refuse(
3250 self,
3251 ctx,
3252 control_error_frame(
3253 &frame,
3254 "busy",
3255 "supervisor list work is at capacity; retry",
3256 )?,
3257 );
3258 return Ok(Vec::new());
3259 };
3260 let handler = self.clone();
3262 let sink = ctx.egress.clone();
3263 tokio::spawn(
3264 async move {
3265 let _permit = permit;
3266 let result = handler.handle_supervisor_list(frame.clone()).await;
3267 let replies = match result {
3268 Ok(replies) => replies,
3269 Err(error) => {
3270 control_error_frame(&frame, "backend_error", error.to_string())
3271 .into_iter()
3272 .collect()
3273 }
3274 };
3275 for reply in replies {
3276 if !matches!(
3277 tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
3278 .await,
3279 Ok(Ok(()))
3280 ) {
3281 break;
3282 }
3283 }
3284 }
3285 .with_current_subscriber(),
3286 );
3287 Ok(Vec::new())
3288 }
3289 ClientControlRequest::SupervisorSpawnSnapshot {} => {
3290 self.handle_supervisor_spawn_snapshot(frame)
3291 }
3292 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
3293 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
3294 }
3295 ClientControlRequest::SupervisorRestart {
3296 module_id,
3297 drain_timeout_ms,
3298 } => {
3299 self.log_supervisor_request_received(
3300 ctx,
3301 frame.header.corr,
3302 ops::SUPERVISOR_RESTART,
3303 Some(&module_id),
3304 None,
3305 )?;
3306 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3307 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
3308 .await
3309 }
3310 ClientControlRequest::SupervisorSwap {
3311 module_id,
3312 ready_timeout_ms,
3313 } => {
3314 self.log_supervisor_request_received(
3315 ctx,
3316 frame.header.corr,
3317 ops::SUPERVISOR_SWAP,
3318 Some(&module_id),
3319 None,
3320 )?;
3321 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3322 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3323 .await
3324 }
3325 ClientControlRequest::SupervisorReload { module_id } => {
3326 self.log_supervisor_request_received(
3327 ctx,
3328 frame.header.corr,
3329 ops::SUPERVISOR_RELOAD,
3330 Some(&module_id),
3331 None,
3332 )?;
3333 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3334 self.handle_supervisor_reload(frame, module_id, requester)
3335 .await
3336 }
3337 ClientControlRequest::SupervisorRescan { preview } => {
3338 if !preview {
3339 self.log_supervisor_request_received(
3340 ctx,
3341 frame.header.corr,
3342 ops::SUPERVISOR_RESCAN,
3343 None,
3344 None,
3345 )?;
3346 }
3347 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3348 let operation = self.handle_supervisor_rescan(
3349 frame,
3350 if preview {
3351 RescanMode::Preview
3352 } else {
3353 RescanMode::Execute
3354 },
3355 requester,
3356 );
3357 self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
3358 .await
3359 }
3360 ClientControlRequest::SupervisorRescanChecked {
3361 preview,
3362 candidate,
3363 expect,
3364 } => {
3365 self.observe_control_wait(
3366 ctx,
3367 None,
3368 ops::SUPERVISOR_RESCAN,
3369 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
3370 )
3371 .await
3372 }
3373 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3374 self.log_supervisor_request_received(
3375 ctx,
3376 frame.header.corr,
3377 ops::SUPERVISOR_RELEASE_RESERVED,
3378 Some(&module_id),
3379 None,
3380 )?;
3381 self.handle_supervisor_release_reserved(frame, module_id)
3382 .await
3383 }
3384 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3385 self.log_supervisor_request_received(
3386 ctx,
3387 frame.header.corr,
3388 ops::SUPERVISOR_SET_ENABLED,
3389 Some(&module_id),
3390 Some(enabled),
3391 )?;
3392 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3393 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3394 .await
3395 }
3396 ClientControlRequest::SupervisorHealthProbe { module_id } => {
3397 self.handle_supervisor_health_probe(ctx, frame, module_id)
3398 .await
3399 }
3400 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3401 ClientControlRequest::SupervisorRoutes { module_id } => {
3402 self.handle_supervisor_routes(frame, module_id)
3403 }
3404 ClientControlRequest::SupervisorProvenance { module_id } => {
3405 self.handle_supervisor_provenance(ctx, frame, module_id)
3406 .await
3407 }
3408 ClientControlRequest::SupervisorStderrTail {
3409 module_id,
3410 max_lines,
3411 max_bytes,
3412 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3413 ClientControlRequest::SupervisorTerminals { module_id } => {
3414 self.handle_supervisor_terminals(ctx, frame, module_id)
3415 .await
3416 }
3417 }
3418 }
3419
3420 fn log_supervisor_request_received(
3421 &self,
3422 ctx: &RouteCtx,
3423 corr: u64,
3424 op: &'static str,
3425 module_id: Option<&str>,
3426 enabled: Option<bool>,
3427 ) -> Result<(), RouterError> {
3428 let caller = self.supervisor_requester(ctx, corr)?;
3429 let caller = principal_label(&caller);
3430 info!(
3433 target: "control",
3434 op,
3435 module_id,
3436 enabled,
3437 connection_id = ctx.connection_id.get(),
3438 caller = %caller,
3439 "supervisor request received"
3440 );
3441 Ok(())
3442 }
3443
3444 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3445 Ok(self
3446 .registry
3447 .get_module_by_connection(ctx.connection_id)
3448 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3449 .map(|registration| Principal::Reserved {
3450 module_id: registration.manifest.module_id,
3451 })
3452 .unwrap_or(Principal::Direct))
3453 }
3454
3455 fn handle_module_control_request(
3456 &self,
3457 connection_id: ConnectionId,
3458 frame: Frame,
3459 request: ModuleControlRequestFromModule,
3460 ) -> Result<Vec<Frame>, RouterError> {
3461 match request {
3462 ModuleControlRequestFromModule::RouteAuthorize {
3463 principal,
3464 scope,
3465 target,
3466 expected,
3467 } => self.handle_route_authorize(
3468 connection_id,
3469 frame,
3470 principal,
3471 scope,
3472 target,
3473 expected,
3474 ),
3475 ModuleControlRequestFromModule::CatalogUpdate {
3476 provides,
3477 capabilities,
3478 ready,
3479 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3480 ModuleControlRequestFromModule::LiveRoots {} => {
3481 let registered = self
3482 .registry
3483 .get_module_by_connection(connection_id)
3484 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3485 let Some(registration) = registered else {
3486 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3487 };
3488 let response = self
3489 .forwarding
3490 .live_roots(®istration.manifest.module_id)
3491 .map_err(RouterError::Forwarding)?;
3492 Ok(vec![control_response_body_frame(
3493 &frame,
3494 &response,
3495 "ModuleControlResponseToModule::LiveRoots",
3496 )?])
3497 }
3498 ModuleControlRequestFromModule::AgentCommands {
3499 started,
3500 finished,
3501 known,
3502 } => self.handle_agent_commands(connection_id, frame, started, finished, known),
3503 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3504 self.handle_scope_sync(connection_id, frame, generation, scopes)
3505 }
3506 ModuleControlRequestFromModule::ScopeApply {
3507 generation,
3508 upsert,
3509 end,
3510 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3511 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3512 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3513 }
3514 _ => Ok(vec![control_error_frame(
3515 &frame,
3516 "unsupported_control_frame",
3517 "unsupported module control operation",
3518 )?]),
3519 }
3520 }
3521
3522 fn handle_scope_served_by(
3523 &self,
3524 connection_id: ConnectionId,
3525 frame: Frame,
3526 ) -> Result<Vec<Frame>, RouterError> {
3527 let registered = self
3528 .registry
3529 .get_module_by_connection(connection_id)
3530 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3531 let Some(registered) = registered else {
3532 return Ok(vec![control_error_frame(
3533 &frame,
3534 "not_registered",
3535 "scope.served_by requires an active module registration owned by this connection",
3536 )?]);
3537 };
3538 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3539 {
3540 Ok(request) => request,
3541 Err(err) => {
3542 return Ok(vec![control_error_frame(
3543 &frame,
3544 "invalid_control_body",
3545 format!("malformed scope.served_by body: {err}"),
3546 )?])
3547 }
3548 };
3549 let answer = self
3550 .scopes
3551 .read()
3552 .unwrap_or_else(|poisoned| poisoned.into_inner())
3553 .served_by(
3554 ®istered.manifest.module_id,
3555 &request.scope_ref,
3556 request.scope_epoch,
3557 &request.module_id,
3558 );
3559 let reply = subc_control::ScopeServedByReply::new(
3560 answer,
3561 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3562 );
3563 Ok(vec![control_response_body_frame(
3564 &frame,
3565 &reply,
3566 "ScopeServedByReply",
3567 )?])
3568 }
3569
3570 fn handle_operator_confirm(
3571 &self,
3572 ctx: &RouteCtx,
3573 frame: Frame,
3574 ) -> Result<Vec<Frame>, RouterError> {
3575 use crate::operator_confirm::{audit, Outcome};
3576 let registration = self
3577 .registry
3578 .get_module_by_connection(ctx.connection_id)
3579 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3580 let Some(registration) = registration else {
3581 let outcome = Outcome::refusal("not_registered");
3582 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3583 return Ok(vec![outcome.frame(&frame)]);
3584 };
3585 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3586 Ok(request) => request,
3587 Err(_) => {
3588 let outcome = Outcome::refusal("invalid_control_body");
3589 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3590 return Ok(vec![outcome.frame(&frame)]);
3591 }
3592 };
3593 let declared = matches!(
3594 request.kind,
3595 Some(subc_protocol::session::OperatorConfirmKind::Operation)
3596 ) && request.operation.as_ref().is_some_and(|operation| {
3597 registration
3598 .manifest
3599 .confirm_operations
3600 .as_ref()
3601 .is_some_and(|operations| operations.contains(operation))
3602 });
3603 let module_id = registration.manifest.module_id;
3604 let nonce = self
3608 .hello_launch_nonces
3609 .lock()
3610 .unwrap_or_else(|p| p.into_inner())
3611 .nonce(ctx.connection_id)
3612 .map(str::to_owned);
3613 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3614 self.supervisor
3615 .spawned_consumer_authorized(&module_id, nonce)
3616 });
3617 let confirms = self.forwarding.operator_confirms();
3618 self.forwarding
3619 .with_operator_route(
3620 ctx.connection_id,
3621 request.route_channel,
3622 request.route_epoch,
3623 |binding| {
3624 let shortcut = binding
3625 .filter(|binding| {
3626 nonce_proven
3627 && crate::operator_confirm::valid_summary(&request.summary)
3628 && declared
3629 && crate::operator_confirm::ShortcutContext::route_eligible(
3630 &self.operator_app_policy,
3631 &self.launch_table,
3632 binding,
3633 )
3634 })
3635 .map(|binding| {
3636 let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
3637 &self.operator_app_policy,
3638 binding,
3639 );
3640 let stamp = if phone.is_some() {
3641 None
3642 } else {
3643 binding.operator_app.clone()
3644 };
3645 self.shortcut_context(
3646 stamp,
3647 phone,
3648 Some((request.route_channel, request.route_epoch)),
3649 )
3650 });
3651 confirms.admit_with_shortcut(
3652 ctx,
3653 frame,
3654 module_id,
3655 nonce_proven,
3656 request,
3657 binding,
3658 shortcut,
3659 )
3660 },
3661 )
3662 .map_err(RouterError::Forwarding)
3663 }
3664
3665 fn shortcut_context(
3666 &self,
3667 stamp: Option<crate::operator_apps::OperatorAppStamp>,
3668 phone_fingerprint: Option<String>,
3669 route: Option<(u16, u32)>,
3670 ) -> crate::operator_confirm::ShortcutContext {
3671 crate::operator_confirm::ShortcutContext {
3672 ledger: self.activity.clone(),
3673 forwarding: self.forwarding.clone(),
3674 launches: self.launch_table.clone(),
3675 stamp,
3676 phone_fingerprint,
3677 route,
3678 connections: self.operator_apps.clone(),
3679 #[cfg(test)]
3680 hooks: Default::default(),
3681 }
3682 }
3683
3684 #[cfg(any(test, feature = "test-support"))]
3686 pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
3687 self.operator_app_policy.phone_enabled = enabled;
3688 self
3689 }
3690
3691 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3694 let mut scopes = ScopeRecords::from(scopes);
3695 for (index, record) in scopes.records.iter_mut().enumerate() {
3696 if let Some(refusal) = Self::canonicalize_workdir(record) {
3697 scopes.refusals.insert(index, refusal);
3698 }
3699 }
3700 scopes
3701 }
3702
3703 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3704 let workdir = record.workdir.as_mut()?;
3705 let path = std::path::Path::new(workdir);
3706 if !path.is_absolute() {
3707 return Some((
3708 error_codes::INVALID_WORKDIR,
3709 "workdir must be absolute".to_string(),
3710 ));
3711 }
3712 match ProjectRootId::from_path_allowing_missing(path) {
3713 Ok(root) => match root.as_path().to_str() {
3714 Some(canonical) => {
3715 *workdir = canonical.to_string();
3716 None
3717 }
3718 None => Some((
3719 error_codes::INVALID_WORKDIR,
3720 "canonical workdir is not UTF-8".to_string(),
3721 )),
3722 },
3723 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3724 }
3725 }
3726
3727 fn handle_scope_sync(
3731 &self,
3732 connection_id: ConnectionId,
3733 frame: Frame,
3734 generation: u64,
3735 scopes: Vec<ScopeRecord>,
3736 ) -> Result<Vec<Frame>, RouterError> {
3737 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3738 }
3739
3740 fn handle_scope_change(
3741 &self,
3742 connection_id: ConnectionId,
3743 frame: Frame,
3744 generation: u64,
3745 scopes: Vec<ScopeRecord>,
3746 end: Option<Vec<ScopeEnd>>,
3747 ) -> Result<Vec<Frame>, RouterError> {
3748 let is_apply = end.is_some();
3749 let Some(registration) = self
3750 .registry
3751 .get_module_by_connection(connection_id)
3752 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3753 else {
3754 return Ok(vec![control_error_frame(
3755 &frame,
3756 "not_registered",
3757 "scope.sync and scope.apply require an active module registration owned by this connection",
3758 )?]);
3759 };
3760 let owner = registration.manifest.module_id;
3761 let scopes = Self::prepare_scope_records(scopes);
3762 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3763 let is_current_launch = |connection: ConnectionId| {
3764 self.hello_launch_nonces
3765 .lock()
3766 .unwrap_or_else(|poisoned| poisoned.into_inner())
3767 .presented(connection, current_nonce.as_deref())
3768 };
3769 let mut table = self
3774 .scopes
3775 .write()
3776 .unwrap_or_else(|poisoned| poisoned.into_inner());
3777 let outcome = match end {
3778 Some(end) => table.apply(
3779 &owner,
3780 connection_id,
3781 is_current_launch,
3782 generation,
3783 scopes,
3784 end,
3785 ),
3786 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3787 };
3788 let drained = match &outcome {
3789 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3790 Err(_) => Vec::new(),
3791 };
3792 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3793 drop(table);
3794 if has_deadlines {
3795 self.scope_deadline_added.notify_one();
3796 }
3797 match outcome {
3798 Ok(applied) => {
3799 let counts = ScopeOutcomeCounts::of(&applied.results);
3800 info!(
3801 owner = %owner,
3802 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3803 generation,
3804 records = applied.results.len(),
3805 created = counts.created,
3806 replaced = counts.replaced,
3807 updated = counts.updated,
3808 unchanged = counts.unchanged,
3809 refused = counts.refused,
3810 ended = applied.ended.len(),
3811 tag_changes = applied.tag_changes.len(),
3812 routes_closed = drained.len(),
3813 "scope change accepted"
3814 );
3815 for refused in applied
3821 .results
3822 .iter()
3823 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3824 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3825 {
3826 warn!(
3827 owner = %owner,
3828 generation,
3829 scope_ref = %refused.scope_ref,
3830 scope_epoch = refused.scope_epoch,
3831 code = refused.code.as_deref().unwrap_or(""),
3832 "scope record refused"
3833 );
3834 }
3835 self.close_scope_drained_routes(drained);
3836 let response = if is_apply {
3837 ModuleControlResponseToModule::ScopeApply {
3838 generation,
3839 results: applied.results,
3840 end_results: applied.end_results,
3841 ended: applied.ended,
3842 }
3843 } else {
3844 ModuleControlResponseToModule::ScopeSync {
3845 generation,
3846 results: applied.results,
3847 ended: applied.ended,
3848 }
3849 };
3850 Ok(vec![control_response_body_frame(
3851 &frame,
3852 &response,
3853 "scope change response",
3854 )?])
3855 }
3856 Err(refusal) => {
3857 info!(
3858 owner = %owner,
3859 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3860 generation,
3861 code = refusal.code,
3862 "scope change refused"
3863 );
3864 Ok(vec![control_error_frame(
3865 &frame,
3866 refusal.code,
3867 refusal.message,
3868 )?])
3869 }
3870 }
3871 }
3872
3873 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3879 if drained.is_empty() {
3880 return;
3881 }
3882 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3883 BTreeMap::new();
3884 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3885 for route in drained {
3886 warn!(
3887 module_id = %route.module_id,
3888 reason = ?route.reason,
3889 client_connection_id = route.client.connection_id.get(),
3890 route_channel = route.client.channel,
3891 "closing route because its scope changed"
3892 );
3893 pushes
3894 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3895 .or_insert_with(|| (route.reason, Vec::new()))
3896 .1
3897 .push(EndpointRoute {
3898 goodbye_target: route.client.clone(),
3899 principal: Principal::Unverified,
3900 bound_at: Instant::now(),
3901 draining: false,
3902 drain_reason: None,
3903 });
3904 goodbyes.push(route.module);
3905 goodbyes.push(route.client);
3906 }
3907 for ((module_id, _), (reason, routes)) in pushes {
3908 send_route_control_pushes(
3909 &self.forwarding,
3910 routes,
3911 ClientControlPush::RouteClosed {
3912 module_id,
3913 channels: Vec::new(),
3914 reason,
3915 drained: false,
3916 abandoned: 0,
3917 excluded_subscriptions: 0,
3918 terminal: Some(false),
3919 },
3920 );
3921 }
3922 self.emit_route_goodbyes(goodbyes);
3923 }
3924
3925 fn handle_route_authorize(
3926 &self,
3927 connection_id: ConnectionId,
3928 frame: Frame,
3929 principal: Principal,
3930 scope: subc_protocol::scope::RouteAuthorizeScope,
3931 target: String,
3932 expected: subc_protocol::scope::RouteAuthorizeExpected,
3933 ) -> Result<Vec<Frame>, RouterError> {
3934 use crate::scopes::admission::{admit_route, AdmissionMode};
3935 use subc_protocol::scope::{
3936 RouteAuthorizeOutcome, RouteAuthorizeResponse, ROUTE_AUTHORIZE_CAPABILITY,
3937 };
3938 let registry = self
3939 .registry
3940 .route_admission_snapshot()
3941 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3942 let owner_configured = match &scope.owner {
3943 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3944 _ => false,
3945 };
3946 let selector = ScopeSelector::new(scope.owner, scope.scope_ref, Some(scope.epoch));
3947 let answer = {
3948 let scopes = self.scopes.read().unwrap_or_else(|p| p.into_inner());
3949 let permitted = registry
3950 .iter()
3951 .find(|r| r.connection_id == connection_id)
3952 .is_some_and(|r| {
3953 scopes.is_federated_owner(&Principal::Reserved {
3954 module_id: r.manifest.module_id.clone(),
3955 }) && r.manifest.capabilities.as_ref().is_some_and(|caps| {
3956 caps.provides
3957 .iter()
3958 .any(|cap| cap == ROUTE_AUTHORIZE_CAPABILITY)
3959 })
3960 });
3961 if !permitted {
3962 drop(scopes);
3963 return Ok(vec![control_error_frame(&frame, error_codes::ROUTE_AUTHORIZE_NOT_PERMITTED,
3964 "route.authorize requires the configured federated owner's route-authorize/v1 registration")?]);
3965 }
3966 admit_route(
3967 &scopes,
3968 ®istry,
3969 &principal,
3970 &target,
3971 Some(&selector),
3972 owner_configured,
3973 AdmissionMode::Authorize(&expected),
3974 )
3975 };
3976 let response = match answer {
3977 Ok(admission) => {
3978 let mut response = RouteAuthorizeResponse::new(
3979 RouteAuthorizeOutcome::Authorized,
3980 "route admission authorized",
3981 );
3982 response.content_version = admission.content_version;
3983 response
3984 }
3985 Err(refusal) => refusal.response(),
3986 };
3987 Ok(vec![control_response_body_frame(
3988 &frame,
3989 &response,
3990 "RouteAuthorizeResponse",
3991 )?])
3992 }
3993
3994 fn handle_scope_describe(
3997 &self,
3998 connection_id: ConnectionId,
3999 frame: Frame,
4000 owner: Principal,
4001 scope_ref: String,
4002 ) -> Result<Vec<Frame>, RouterError> {
4003 let registered = self
4004 .registry
4005 .get_module_by_connection(connection_id)
4006 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4007 let Some(registered) = registered else {
4008 return Ok(vec![control_error_frame(
4009 &frame,
4010 "not_registered",
4011 "scope.describe requires an active module registration owned by this connection",
4012 )?]);
4013 };
4014 let scopes = self
4015 .scopes
4016 .read()
4017 .unwrap_or_else(|poisoned| poisoned.into_inner());
4018 if scopes.is_federated_owner(&owner)
4019 && !crate::scopes::target_supports_federated(registered.manifest.capabilities.as_ref())
4020 {
4021 return Ok(vec![control_error_frame(
4022 &frame,
4023 error_codes::SCOPE_FEDERATED_UNSUPPORTED,
4024 "scope.describe of the federated owner requires scope-federated/v1",
4025 )?]);
4026 }
4027 let mut description = scopes.describe(&owner, &scope_ref);
4028 drop(scopes);
4029 if owner
4030 != (Principal::Reserved {
4031 module_id: registered.manifest.module_id.clone(),
4032 })
4033 {
4034 if let Some(facts) = description
4035 .stamp
4036 .as_mut()
4037 .and_then(|stamp| stamp.attributes.federated.as_mut())
4038 {
4039 facts.stamp_required = false;
4040 }
4041 }
4042 if !registered
4045 .manifest
4046 .capabilities
4047 .as_ref()
4048 .is_some_and(|caps| {
4049 caps.provides
4050 .iter()
4051 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
4052 })
4053 {
4054 if let Some(stamp) = description.stamp.as_mut() {
4055 stamp.attributes.on_behalf_of_agent = None;
4056 }
4057 }
4058 if !registered
4059 .manifest
4060 .capabilities
4061 .as_ref()
4062 .is_some_and(|caps| {
4063 caps.provides
4064 .iter()
4065 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
4066 })
4067 {
4068 if let Some(stamp) = description.stamp.as_mut() {
4069 stamp.attributes.flow_version = None;
4070 stamp.attributes.flow_code_hash = None;
4071 stamp.attributes.flow_install_id = None;
4072 }
4073 }
4074 let owner_configured = match &owner {
4075 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4080 _ => false,
4081 };
4082 let response = ModuleControlResponseToModule::ScopeDescribe {
4083 status: description.status,
4084 scope_epoch: description.scope_epoch,
4085 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
4086 owner_synced: description.owner_synced,
4087 owner_configured,
4088 scope: description.stamp,
4089 };
4090 Ok(vec![control_response_body_frame(
4091 &frame,
4092 &response,
4093 "ModuleControlResponseToModule::ScopeDescribe",
4094 )?])
4095 }
4096
4097 fn handle_catalog_update(
4098 &self,
4099 connection_id: ConnectionId,
4100 frame: Frame,
4101 provides: Vec<ProviderRole>,
4102 capabilities: Option<CapabilityDeclarations>,
4103 ready: Option<bool>,
4104 ) -> Result<Vec<Frame>, RouterError> {
4105 self.refresh_capability_requirements();
4106 let Some(registration) = self
4107 .registry
4108 .get_module_by_connection(connection_id)
4109 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4110 else {
4111 return Ok(vec![control_error_frame(
4112 &frame,
4113 "not_registered",
4114 "catalog.update requires an active module registration owned by this connection",
4115 )?]);
4116 };
4117
4118 if let Some(message) =
4119 catalog_update_frozen_field_message(®istration.manifest, &provides)
4120 {
4121 return Ok(vec![control_error_frame(
4122 &frame,
4123 "catalog_update_frozen_field",
4124 message,
4125 )?]);
4126 }
4127
4128 let mut candidate = registration.manifest.clone();
4129 candidate.provides = provides.clone();
4130 candidate.capabilities = capabilities
4131 .clone()
4132 .or_else(|| registration.manifest.capabilities.clone());
4133 if let Some(rich_ui) = &candidate.rich_ui {
4134 if let Err(err) = rich_ui.validate() {
4135 return Ok(vec![control_error_frame(
4136 &frame,
4137 "invalid_manifest",
4138 err.to_string(),
4139 )?]);
4140 }
4141 }
4142 if let Err(err) = candidate.validate_capability_grammar() {
4143 return Ok(vec![control_error_frame(
4144 &frame,
4145 "invalid_capability_grammar",
4146 err.to_string(),
4147 )?]);
4148 }
4149
4150 let mut conflicts = self
4153 .capability_evaluator
4154 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
4155 if let Some(conflict) = conflicts.first() {
4156 let message = format!(
4157 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
4158 conflict.capability, conflict.claimants[0], candidate.module_id
4159 );
4160 for conflict in &mut conflicts {
4161 conflict.source = DuplicateClaimSource::CatalogUpdate;
4162 }
4163 log_duplicate_claim_events(conflicts);
4164 return Ok(vec![control_error_frame(
4165 &frame,
4166 "reserved_capability",
4167 message,
4168 )?]);
4169 }
4170
4171 let updated = match self.registry.replace_catalog_for_connection(connection_id, provides, capabilities, ready) {
4172 Err(crate::registry::RegistryError::FederatedCapabilityChanged) => return Ok(vec![control_error_frame(
4173 &frame, error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
4174 "scope-federated/v1 is fixed at HELLO; re-register on a new connection to change it",
4175 )?]),
4176 result => result.map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?,
4177 };
4178 if updated.is_none() {
4179 return Ok(vec![control_error_frame(
4180 &frame,
4181 "not_registered",
4182 "catalog.update requires an active module registration owned by this connection",
4183 )?]);
4184 }
4185 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
4186 log_duplicate_claim_events(
4187 self.capability_evaluator
4188 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
4189 );
4190 }
4191 if capability_census_trigger(
4192 registration.manifest.capabilities.as_ref(),
4193 updated
4194 .as_ref()
4195 .and_then(|entry| entry.manifest.capabilities.as_ref()),
4196 ) {
4197 self.enforce_capability_denies();
4198 }
4199 self.refresh_capability_requirements();
4200
4201 let response = ModuleControlResponseToModule::CatalogUpdate {};
4202 control_response_body_frame(
4203 &frame,
4204 &response,
4205 "ModuleControlResponseToModule::CatalogUpdate",
4206 )
4207 .map(|frame| vec![frame])
4208 }
4209
4210 pub(crate) fn with_activity_ledger_config(
4211 mut self,
4212 run_dir: &Path,
4213 incarnation: String,
4214 config: crate::activity::ActivityConfig,
4215 ) -> Self {
4216 self.activity = Some(crate::activity::ActivityLedger::open_config(
4217 run_dir.join("activity.jsonl"),
4218 incarnation,
4219 config,
4220 ));
4221 self.supervisor.set_activity(self.activity.clone());
4222 self
4223 }
4224
4225 pub(crate) async fn configure_activity(
4226 &self,
4227 config: crate::activity::ActivityConfig,
4228 ) -> Result<(), &'static str> {
4229 match &self.activity {
4230 Some(ledger) => ledger.configure(config).await,
4231 None => Ok(()),
4232 }
4233 }
4234
4235 fn cancel_activity_subscription(
4236 &self,
4237 ctx: &RouteCtx,
4238 frame: &Frame,
4239 ) -> Result<bool, RouterError> {
4240 let task = self
4241 .activity_streams
4242 .lock()
4243 .unwrap_or_else(|p| p.into_inner())
4244 .remove(&(ctx.connection_id, frame.header.corr));
4245 let Some(task) = task else {
4246 return Ok(false);
4247 };
4248 task.abort();
4249 let end = Frame::build_with_version(
4250 response_version(frame),
4251 FrameType::StreamEnd,
4252 control_flags(),
4253 0,
4254 0,
4255 frame.header.corr,
4256 Vec::new(),
4257 )
4258 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
4259 let sink = ctx.egress.clone();
4260 tokio::spawn(async move {
4261 let _ = sink.send(end).await;
4262 });
4263 Ok(true)
4264 }
4265
4266 async fn handle_activity_read(
4267 &self,
4268 ctx: &RouteCtx,
4269 frame: Frame,
4270 ) -> Result<Vec<Frame>, RouterError> {
4271 use subc_control::{
4272 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
4273 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
4274 };
4275 #[derive(Deserialize)]
4276 struct Identity {
4277 #[serde(default)]
4278 consumer_identity: Option<ConsumerIdentity>,
4279 }
4280 let principal = match self
4284 .forwarding
4285 .module_id_for_connection(ctx.connection_id)
4286 .map_err(RouterError::Forwarding)?
4287 {
4288 Some(module_id) => Ok(Principal::Reserved { module_id }),
4289 None => {
4290 let identity: Identity = match serde_json::from_slice(&frame.body) {
4291 Ok(identity) => identity,
4292 Err(_) => {
4293 return Ok(vec![control_error_frame(
4294 &frame,
4295 ACTIVITY_MALFORMED,
4296 "malformed activity read",
4297 )?])
4298 }
4299 };
4300 self.route_open_principal(&frame, identity.consumer_identity)?
4301 }
4302 };
4303 let admitted = matches!(principal, Ok(Principal::Direct))
4304 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
4305 if !admitted {
4306 return Ok(vec![control_error_frame(
4307 &frame,
4308 ACTIVITY_READ_FORBIDDEN,
4309 "activity reads require an operator or trusted mirror principal",
4310 )?]);
4311 }
4312 let Some(ledger) = &self.activity else {
4313 return Ok(vec![control_error_frame(
4314 &frame,
4315 ACTIVITY_UNAVAILABLE,
4316 "activity ledger unavailable",
4317 )?]);
4318 };
4319 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
4320 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
4321 {
4322 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
4323 Ok(request) => ledger.query(request).await,
4324 Err(_) => Err(ACTIVITY_MALFORMED),
4325 };
4326 return match result {
4327 Ok(reply) => Ok(vec![control_response_body_frame(
4328 &frame,
4329 &reply,
4330 ops::ACTIVITY_QUERY,
4331 )?]),
4332 Err(code) => Ok(vec![control_error_frame(
4333 &frame,
4334 code,
4335 "activity query refused",
4336 )?]),
4337 };
4338 }
4339 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
4340 Ok(request) => request,
4341 Err(_) => {
4342 return Ok(vec![control_error_frame(
4343 &frame,
4344 ACTIVITY_MALFORMED,
4345 "malformed activity subscription",
4346 )?])
4347 }
4348 };
4349 let mut subscription = match ledger.subscribe(request.since).await {
4350 Ok(subscription) => subscription,
4351 Err(code) => {
4352 return Ok(vec![control_error_frame(
4353 &frame,
4354 code,
4355 "activity subscription refused",
4356 )?])
4357 }
4358 };
4359 let key = (ctx.connection_id, frame.header.corr);
4360 let sink = ctx.egress.clone();
4361 let mut streams = self
4362 .activity_streams
4363 .lock()
4364 .unwrap_or_else(|p| p.into_inner());
4365 streams.retain(|_, task| !task.is_finished());
4366 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
4367 return Ok(vec![control_error_frame(
4368 &frame,
4369 "duplicate_subscription",
4370 "activity subscription correlation id already in use",
4371 )?]);
4372 }
4373 let task = tokio::spawn(async move {
4374 loop {
4375 let entry = if let Some(entry) = subscription.replay.pop_front() {
4376 Some(entry)
4377 } else {
4378 subscription.receiver.recv().await
4379 };
4380 let Some(entry) = entry else {
4381 break;
4382 };
4383 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
4384 return;
4385 };
4386 let Ok(next) = Frame::build_with_version(
4387 response_version(&frame),
4388 FrameType::StreamData,
4389 control_flags(),
4390 0,
4391 0,
4392 frame.header.corr,
4393 body,
4394 ) else {
4395 return;
4396 };
4397 if sink.send(next).await.is_err() {
4398 return;
4399 }
4400 }
4401 if let Some(code) = subscription.terminal_code() {
4404 if let Ok(end) =
4405 control_error_frame(&frame, code, "activity subscriber fell behind")
4406 {
4407 let _ = sink.send(end).await;
4408 }
4409 }
4410 });
4411 streams.insert(key, task.abort_handle());
4412 Ok(Vec::new())
4413 }
4414
4415 async fn handle_activity_record(
4416 &self,
4417 ctx: &RouteCtx,
4418 frame: Frame,
4419 ) -> Result<Vec<Frame>, RouterError> {
4420 use subc_control::{
4421 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
4422 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
4423 };
4424 let result = async {
4425 let module = self
4426 .forwarding
4427 .module_id_for_connection(ctx.connection_id)
4428 .ok()
4429 .flatten()
4430 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
4431 let request: ActivityRecordRequest =
4432 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
4433 crate::activity::validate(&module, &request)?;
4434 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
4435 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
4436 let at_ms = std::time::SystemTime::now()
4437 .duration_since(std::time::UNIX_EPOCH)
4438 .unwrap_or_default()
4439 .as_millis() as u64;
4440 ledger.record(module, request, actor, at_ms).await
4441 }
4442 .await;
4443 match result {
4444 Ok(entry) => Ok(vec![control_response_body_frame(
4445 &frame,
4446 &ActivityRecordResponse { seq: entry.seq },
4447 "activity.record",
4448 )?]),
4449 Err(code) => Ok(vec![control_error_frame(
4450 &frame,
4451 code,
4452 "activity recording refused",
4453 )?]),
4454 }
4455 }
4456
4457 fn activity_actor(
4458 &self,
4459 connection: ConnectionId,
4460 module: &str,
4461 on: Option<&subc_control::ActivityOn>,
4462 ) -> Result<subc_control::ActivityActor, &'static str> {
4463 use crate::forwarding::{DataRoute, DataRouteState};
4464 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
4465 let principal = |id: &str| Principal::Reserved {
4466 module_id: id.to_owned(),
4467 };
4468 let mut actor = ActivityActor {
4469 principal: principal(module),
4470 scope_owner: None,
4471 agent_id: None,
4472 run_id: None,
4473 flow_id: None,
4474 project_root: None,
4475 workdir: None,
4476 project_id: None,
4477 };
4478 let scope = match on {
4479 None => return Ok(actor),
4480 Some(ActivityOn::Scope(scope)) => {
4481 let selector = subc_protocol::scope::ScopeSelector::new(
4482 principal(module),
4483 scope.scope_ref.clone(),
4484 Some(scope.epoch),
4485 );
4486 self.scopes
4487 .read()
4488 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4489 .attribute(&principal(module), module, &selector, true)
4490 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4491 }
4492 Some(ActivityOn::Route(route)) => {
4493 let DataRoute::Module(DataRouteState::Bound(binding)) = self
4494 .forwarding
4495 .lookup_data_route(connection, route.channel, route.epoch)
4496 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4497 else {
4498 return Err(ACTIVITY_ACTOR_UNKNOWN);
4499 };
4500 if binding.module_id != module {
4501 return Err(ACTIVITY_ACTOR_UNKNOWN);
4502 }
4503 actor.principal = binding.principal.clone();
4504 actor.project_root = binding
4505 .project_root
4506 .as_ref()
4507 .map(|root| root.as_path().to_path_buf());
4508 let Some(bound) = &binding.scope else {
4509 return Ok(actor);
4510 };
4511 let owner = principal(&bound.owner);
4512 let selector = subc_protocol::scope::ScopeSelector::new(
4513 owner.clone(),
4514 bound.scope_ref.clone(),
4515 Some(bound.tag.scope_epoch),
4516 );
4517 let admitted = self
4518 .scopes
4519 .read()
4520 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4521 .attribute(&owner, module, &selector, true)
4522 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4523 if admitted.tag != bound.tag {
4524 return Err(ACTIVITY_ACTOR_UNKNOWN);
4525 }
4526 admitted
4527 }
4528 };
4529 actor.scope_owner = Some(scope.stamp.owner);
4530 actor.agent_id = scope.stamp.attributes.agent_id;
4531 actor.run_id = scope.stamp.attributes.run_id;
4532 actor.flow_id = scope.stamp.attributes.flow_id;
4533 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4534 actor.project_id = scope.stamp.project_id;
4535 Ok(actor)
4536 }
4537
4538 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4539 self.refresh_capability_requirements();
4540 let mut counters = self.counters.snapshot();
4546 let losses = self.resource_tracker.loss_diagnostics();
4547 if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
4548 if let Some(obj) = counters.as_object_mut() {
4549 obj.insert("agent_command_declaration_loss_by_module".into(), losses);
4550 }
4551 }
4552 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4553 obj.insert(
4554 "activity_unparseable_lines".into(),
4555 ledger.read_errors().into(),
4556 );
4557 obj.insert(
4558 "activity_lifecycle_append_failures".into(),
4559 ledger.lifecycle_failures().into(),
4560 );
4561 obj.insert(
4562 "activity_started_digest_skips".into(),
4563 ledger.lifecycle_digest_skips().into(),
4564 );
4565 }
4566 if let (Ok((connections_with_routes, max)), Some(obj)) = (
4567 self.forwarding.client_route_concentration(),
4568 counters.as_object_mut(),
4569 ) {
4570 obj.insert(
4571 "client_connections_with_routes".into(),
4572 connections_with_routes.into(),
4573 );
4574 obj.insert("max_routes_on_one_connection".into(), max.into());
4575 }
4576 if let (Some(open_breakers), Some(obj)) = (
4582 self.route_bind_breakers.open_snapshot(),
4583 counters.as_object_mut(),
4584 ) {
4585 obj.insert("route_bind_breakers_open".into(), open_breakers);
4586 }
4587 if let (Some(reason), Some(obj)) = (
4592 self.local_endpoint_unavailable.clone(),
4593 counters.as_object_mut(),
4594 ) {
4595 obj.insert("local_endpoint_unavailable".into(), reason.into());
4596 }
4597 let response = ClientControlResponse::ServerDescribe {
4598 protocol_ver: PROTOCOL_VERSION,
4599 subc_ops: subc_ops(),
4600 capabilities: self.describe_capabilities(),
4601 connected_clients: self.connected_clients.count(),
4602 counters: Some(counters),
4603 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4604 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4605 capability_requirements: self.capability_requirement_statuses(),
4606 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4607 config_path: self
4608 .rescan
4609 .as_ref()
4610 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4611 };
4612 Ok(vec![control_response_body_frame(
4613 &frame,
4614 &response,
4615 "ClientControlResponse::ServerDescribe",
4616 )?])
4617 }
4618
4619 fn handle_catalog_list(
4620 &self,
4621 frame: Frame,
4622 module_id: Option<String>,
4623 ) -> Result<Vec<Frame>, RouterError> {
4624 let (generation, modules) = self
4625 .registry
4626 .list_modules_with_nonce_verification()
4627 .map_err(|err| {
4628 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4629 })?;
4630 let entries = modules
4631 .into_iter()
4632 .filter(|(registration, _)| {
4633 module_id
4634 .as_deref()
4635 .map(|wanted| registration.manifest.module_id == wanted)
4636 .unwrap_or(true)
4637 })
4638 .map(|(registration, nonce_verified)| {
4639 let not_ready = self.not_ready_reason(®istration);
4640 let roles = registration.manifest.provides;
4641 CatalogEntry::new(
4642 registration.manifest.module_id,
4643 roles,
4644 registration.control_ops,
4645 )
4646 .with_ready(not_ready.is_none())
4647 .with_display_name(registration.manifest.display_name)
4648 .with_not_ready(not_ready)
4649 .with_module_version(Some(registration.manifest.module_version))
4650 .with_provenance(registration.manifest.provenance)
4651 .with_capabilities(registration.manifest.capabilities)
4652 .with_self_signals(registration.manifest.self_signals)
4653 .with_events(registration.manifest.events)
4654 .with_operation_delivery(registration.manifest.operation_delivery)
4655 .with_operation_payload(registration.manifest.operation_payload)
4656 .with_flow_operations(registration.manifest.flow_operations)
4657 .with_operation_deferral(registration.manifest.operation_deferral)
4658 .with_confirm_operations(registration.manifest.confirm_operations)
4659 .with_rich_ui(registration.manifest.rich_ui)
4660 .with_role_declarations(registration.manifest.role_declarations)
4661 .with_nonce_verified(nonce_verified)
4662 })
4663 .collect();
4664 let response = ClientControlResponse::CatalogList {
4665 generation,
4666 modules: entries,
4667 subc_ops: subc_ops(),
4668 };
4669 Ok(vec![control_response_body_frame(
4670 &frame,
4671 &response,
4672 "ClientControlResponse::CatalogList",
4673 )?])
4674 }
4675
4676 fn attest_route_open_consumer(
4681 &self,
4682 consumer_identity: Option<&ConsumerIdentity>,
4683 ) -> Option<Principal> {
4684 let Some(consumer_identity) = consumer_identity else {
4685 return Some(Principal::Direct);
4686 };
4687 self.supervisor
4688 .spawned_consumer_authorized(
4689 &consumer_identity.module_id,
4690 &consumer_identity.launch_nonce,
4691 )
4692 .then(|| Principal::Reserved {
4693 module_id: consumer_identity.module_id.clone(),
4694 })
4695 }
4696
4697 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4701 let Ok(ClientControlRequest::RouteOpen {
4702 identity,
4703 consumer_identity,
4704 scope,
4705 ..
4706 }) = parse_client_control_request(&frame.body)
4707 else {
4708 return RouteOpenCaller::unknown();
4709 };
4710 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4711 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4712 }
4713
4714 fn route_open_principal(
4715 &self,
4716 frame: &Frame,
4717 consumer_identity: Option<ConsumerIdentity>,
4718 ) -> Result<Result<Principal, Frame>, RouterError> {
4719 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4720 self.route_open_principal_attested(frame, consumer_identity, attested)
4721 }
4722
4723 fn route_open_principal_attested(
4727 &self,
4728 frame: &Frame,
4729 consumer_identity: Option<ConsumerIdentity>,
4730 attested: Option<Principal>,
4731 ) -> Result<Result<Principal, Frame>, RouterError> {
4732 if let Some(principal) = attested {
4733 return Ok(Ok(principal));
4734 }
4735 let Some(consumer_identity) = consumer_identity else {
4736 return Ok(Ok(Principal::Direct));
4737 };
4738
4739 Ok(Err(control_error_frame(
4740 frame,
4741 "bad_consumer_identity",
4742 format!(
4743 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4744 consumer_identity.module_id
4745 ),
4746 )?))
4747 }
4748
4749 #[allow(clippy::too_many_arguments)]
4756 fn route_open_refusal_frame(
4757 &self,
4758 ctx: &RouteCtx,
4759 caller: &RouteOpenCaller,
4760 frame: &Frame,
4761 module_id: &str,
4762 reason: &'static str,
4763 code: &'static str,
4764 message: impl Into<String>,
4765 ) -> Result<Frame, RouterError> {
4766 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4767 control_error_frame(frame, code, message.into())
4768 }
4769
4770 #[allow(clippy::too_many_arguments)]
4779 fn route_bind_rejection(
4780 &self,
4781 ctx: &RouteCtx,
4782 caller: &RouteOpenCaller,
4783 frame: &Frame,
4784 target_module_id: &str,
4785 body: ErrorBody,
4786 daemon_authored: bool,
4787 ) -> Result<Vec<Frame>, RouterError> {
4788 if daemon_authored {
4794 let scope_code = match body.code.as_str() {
4795 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4796 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4797 _ => None,
4798 };
4799 if let Some(code) = scope_code {
4800 self.observe_route_open_refusal(
4801 ctx,
4802 caller,
4803 target_module_id,
4804 "scope_changed_before_commit",
4805 code,
4806 );
4807 return Ok(vec![control_error_body_frame(frame, body)?]);
4808 }
4809 }
4810 self.counters
4811 .increment_route_open_refused("module_rejected");
4812 info!(
4813 target: "control",
4814 code = "module_rejected",
4815 module_code = ?body.code,
4816 module_id = ?target_module_id,
4817 connection_id = ctx.connection_id.get(),
4818 principal = %caller.principal,
4819 scope = %caller.scope,
4820 harness = %caller.harness,
4821 "route.open refused"
4822 );
4823 let refusal = if daemon_authored {
4824 control_error_body_frame(frame, body)?
4825 } else {
4826 control_relayed_error_body_frame(frame, body)?
4827 };
4828 Ok(vec![refusal])
4829 }
4830
4831 #[allow(clippy::too_many_arguments)]
4849 fn route_open_breaker_refusal_frame(
4850 &self,
4851 ctx: &RouteCtx,
4852 caller: &RouteOpenCaller,
4853 frame: &Frame,
4854 module_id: &str,
4855 consecutive_timeouts: u32,
4856 retry_in: Duration,
4857 probe_in_flight: bool,
4858 ) -> Result<Frame, RouterError> {
4859 self.counters
4860 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4861 debug!(
4862 target: "control",
4863 code = "module_timeout",
4864 module_id = ?module_id,
4865 connection_id = ctx.connection_id.get(),
4866 principal = %caller.principal,
4867 scope = %caller.scope,
4868 harness = %caller.harness,
4869 consecutive_timeouts,
4870 retry_in_ms = retry_in.as_millis() as u64,
4871 probe_in_flight,
4872 "route.open refused by open bind-relay breaker"
4873 );
4874 let detail = if probe_in_flight {
4881 "one test route is already being tried; retry once it settles".to_string()
4882 } else {
4883 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4884 };
4885 control_error_frame(
4886 frame,
4887 "module_timeout",
4888 format!(
4889 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4890 timed out in a row); {detail}; its established routes are unaffected"
4891 ),
4892 )
4893 }
4894
4895 fn observe_route_open_refusal(
4908 &self,
4909 ctx: &RouteCtx,
4910 caller: &RouteOpenCaller,
4911 module_id: &str,
4912 reason: &'static str,
4913 code: &'static str,
4914 ) {
4915 self.counters.increment_route_open_refused(code);
4916 info!(
4917 target: "control",
4918 code,
4919 reason,
4920 module_id = ?module_id,
4921 connection_id = ctx.connection_id.get(),
4922 principal = %caller.principal,
4923 scope = %caller.scope,
4924 harness = %caller.harness,
4925 "route.open refused"
4926 );
4927 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4928 self.route_outages.record_not_serving(module_id, reason);
4929 }
4930 }
4931
4932 fn observe_route_open_accept(
4977 &self,
4978 ctx: &RouteCtx,
4979 module_id: &str,
4980 principal: &str,
4981 scope: &str,
4982 ) {
4983 self.route_outages.record_accepted(module_id);
4984 self.counters.increment_route_open_accepted(principal);
4985 info!(
4986 target: "control",
4987 principal,
4988 module_id,
4989 connection_id = ctx.connection_id.get(),
4990 scope = %scope,
4991 "route.open accepted"
4992 );
4993 }
4994
4995 fn supervised_absent_route_open_refusal_frame(
4996 &self,
4997 ctx: &RouteCtx,
4998 caller: &RouteOpenCaller,
4999 frame: &Frame,
5000 module_id: &str,
5001 code: &'static str,
5002 status: &crate::supervise::ModuleStatus,
5003 ) -> Result<Frame, RouterError> {
5004 self.counters.increment_route_open_refused(code);
5005 info!(
5006 target: "control",
5007 code,
5008 reason = "supervised_not_registered",
5009 module_id = ?module_id,
5010 connection_id = ctx.connection_id.get(),
5011 principal = %caller.principal,
5012 scope = %caller.scope,
5013 harness = %caller.harness,
5014 state = %status.state,
5015 enabled = status.enabled,
5016 live = status.live,
5017 "route.open refused"
5018 );
5019 self.route_outages
5023 .record_not_serving(module_id, "supervised_not_registered");
5024 let message = if let Some(pid) = status.waiting_for_previous_pid {
5025 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
5026 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
5027 })
5028 } else {
5029 format!(
5030 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
5031 status.state, status.enabled, status.live
5032 )
5033 };
5034 control_error_frame(frame, code, message)
5035 }
5036
5037 async fn handle_route_open(
5038 &self,
5039 ctx: &RouteCtx,
5040 frame: Frame,
5041 request: RouteOpenRequest,
5042 timings: &mut RouteOpenTimings,
5043 ) -> Result<Vec<Frame>, RouterError> {
5044 let RouteOpenRequest {
5045 target,
5046 mut identity,
5047 consumer_identity,
5048 consumer_capabilities,
5049 role_versions,
5050 busy_refusals,
5051 admission_facts,
5052 scope,
5053 } = request;
5054 let target_module_id = target_module_id(&target).to_string();
5055 timings.module_id.clone_from(&target_module_id);
5056 if self
5057 .registry
5058 .get_module_by_connection(ctx.connection_id)
5059 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5060 .is_some()
5061 {
5062 return Ok(vec![control_error_frame(
5063 &frame,
5064 "invalid_request",
5065 "module connections cannot open client routes",
5066 )?]);
5067 }
5068 debug!(
5069 connection_id = ctx.connection_id.get(),
5070 corr = frame.header.corr,
5071 module_id = %target_module_id,
5072 "handling route.open"
5073 );
5074
5075 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
5080 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
5081
5082 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
5088 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
5089 self.observe_route_open_refusal(
5090 ctx,
5091 &caller,
5092 &target_module_id,
5093 "invalid_role_versions",
5094 error_codes::INVALID_REQUEST,
5095 );
5096 return Ok(vec![control_error_body_frame(
5097 &frame,
5098 ErrorBody {
5099 code: error_codes::INVALID_REQUEST.to_string(),
5100 message: error.to_string(),
5101 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
5102 },
5103 )?]);
5104 }
5105
5106 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
5107 if status.waiting_for_previous_pid.is_some() {
5108 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
5109 ctx,
5110 &caller,
5111 &frame,
5112 &target_module_id,
5113 "module_warming",
5114 &status,
5115 )?]);
5116 }
5117 }
5118
5119 let admission_registry = self
5138 .registry
5139 .route_admission_snapshot()
5140 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5141 let Some(registration) =
5142 crate::scopes::admission::registered_target(&admission_registry, &target_module_id)
5143 else {
5144 if let Some((status, warming)) =
5145 self.supervisor_status(&target_module_id, frame.header.corr)?
5146 {
5147 if status.protocol == ModuleProtocol::None {
5154 return Ok(vec![self.route_open_refusal_frame(
5155 ctx,
5156 &caller,
5157 &frame,
5158 &target_module_id,
5159 "protocol_none",
5160 error_codes::MODULE_NO_PROTOCOL,
5161 format!(
5162 "module_id '{target_module_id}' is declared protocol: none; \
5163 it speaks no subc wire and serves no routes"
5164 ),
5165 )?]);
5166 }
5167 let code = if warming {
5168 "module_warming"
5169 } else {
5170 "target_unavailable"
5171 };
5172 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
5173 ctx,
5174 &caller,
5175 &frame,
5176 &target_module_id,
5177 code,
5178 &status,
5179 )?]);
5180 }
5181 if let Some(removed_ago_ms) =
5182 self.supervisor.removal_tombstone_age_ms(&target_module_id)
5183 {
5184 return Ok(vec![self.route_open_refusal_frame(
5185 ctx,
5186 &caller,
5187 &frame,
5188 &target_module_id,
5189 "removed",
5190 error_codes::MODULE_REMOVED,
5191 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
5192 )?]);
5193 }
5194 return Ok(vec![self.route_open_refusal_frame(
5195 ctx,
5196 &caller,
5197 &frame,
5198 &target_module_id,
5199 "not_registered",
5200 error_codes::UNKNOWN_MODULE,
5201 format!("module_id '{target_module_id}' is not registered"),
5202 )?]);
5203 };
5204
5205 if crate::scopes::admission::target_registration(&admission_registry, &target_module_id)
5209 .is_err()
5210 {
5211 self.counters
5212 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
5213 info!(
5214 target: "control",
5215 code = error_codes::MODULE_WARMING,
5216 module_id = ?target_module_id,
5217 connection_id = ctx.connection_id.get(),
5218 principal = %caller.principal,
5219 scope = %caller.scope,
5220 harness = %caller.harness,
5221 reason = "declared_not_ready",
5222 "route.open refused"
5223 );
5224 self.route_outages
5227 .record_not_serving(&target_module_id, "declared_not_ready");
5228 return Ok(vec![control_error_body_frame(
5229 &frame,
5230 ErrorBody {
5231 code: error_codes::MODULE_WARMING.to_string(),
5232 message: format!(
5233 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
5234 ),
5235 detail: Some(serde_json::json!({
5236 "reason": "declared_not_ready"
5237 })),
5238 },
5239 )?]);
5240 }
5241
5242 if let Some(capability) = self.first_unprovided_required_capability(registration) {
5261 self.counters
5262 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
5263 info!(
5264 target: "control",
5265 code = error_codes::MODULE_WARMING,
5266 module_id = ?target_module_id,
5267 connection_id = ctx.connection_id.get(),
5268 principal = %caller.principal,
5269 scope = %caller.scope,
5270 harness = %caller.harness,
5271 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5272 capability = %capability,
5273 "route.open refused"
5274 );
5275 return Ok(vec![control_error_body_frame(
5276 &frame,
5277 ErrorBody {
5278 code: error_codes::MODULE_WARMING.to_string(),
5279 message: format!(
5280 "module_id '{target_module_id}' requires capability '{capability}', \
5281 which no registered module provides; retry"
5282 ),
5283 detail: Some(serde_json::json!({
5284 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5285 "capability": capability,
5286 })),
5287 },
5288 )?]);
5289 }
5290
5291 if !target_has_required_role(&target, ®istration.manifest.provides) {
5292 return Ok(vec![self.route_open_refusal_frame(
5293 ctx,
5294 &caller,
5295 &frame,
5296 &target_module_id,
5297 "role_not_provided",
5298 "target_unavailable",
5299 format!("module_id '{target_module_id}' does not provide the requested target"),
5300 )?]);
5301 }
5302
5303 if registration.state != ChannelState::Active {
5304 return Ok(vec![self.route_open_refusal_frame(
5305 ctx,
5306 &caller,
5307 &frame,
5308 &target_module_id,
5309 "registration_not_active",
5310 "target_unavailable",
5311 format!("module_id '{target_module_id}' is not active"),
5312 )?]);
5313 }
5314
5315 if self
5316 .forwarding
5317 .module_is_draining(&target_module_id)
5318 .map_err(RouterError::Forwarding)?
5319 {
5320 return Ok(vec![self.route_open_refusal_frame(
5321 ctx,
5322 &caller,
5323 &frame,
5324 &target_module_id,
5325 "reloading",
5326 "module_reloading",
5327 format!("module_id '{target_module_id}' is reloading"),
5328 )?]);
5329 }
5330
5331 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
5332 process_liveness.process_live(&target_module_id) == Some(false)
5333 }) {
5334 if process_liveness.process_replacing(&target_module_id) {
5341 return Ok(vec![self.route_open_refusal_frame(
5342 ctx,
5343 &caller,
5344 &frame,
5345 &target_module_id,
5346 "reloading",
5347 "module_reloading",
5348 format!("module_id '{target_module_id}' is reloading"),
5349 )?]);
5350 }
5351 return Ok(vec![self.route_open_refusal_frame(
5352 ctx,
5353 &caller,
5354 &frame,
5355 &target_module_id,
5356 "supervisor_not_live",
5357 "target_unavailable",
5358 format!("module_id '{target_module_id}' is not live"),
5359 )?]);
5360 }
5361
5362 if !self
5363 .forwarding
5364 .has_live_module_connection(&target_module_id)
5365 .map_err(RouterError::Forwarding)?
5366 {
5367 return Ok(vec![self.route_open_refusal_frame(
5368 ctx,
5369 &caller,
5370 &frame,
5371 &target_module_id,
5372 "no_forwarding_connection",
5373 "target_unavailable",
5374 format!("module_id '{target_module_id}' has no live forwarding connection"),
5375 )?]);
5376 }
5377
5378 if let Some(error) =
5379 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
5380 {
5381 self.observe_route_open_refusal(
5382 ctx,
5383 &caller,
5384 &target_module_id,
5385 "op_not_allowed",
5386 "op_not_allowed",
5387 );
5388 return Ok(vec![error]);
5389 }
5390
5391 let principal =
5392 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
5393 Ok(principal) => principal,
5394 Err(error) => {
5395 self.observe_route_open_refusal(
5396 ctx,
5397 &caller,
5398 &target_module_id,
5399 "bad_consumer_identity",
5400 "bad_consumer_identity",
5401 );
5402 return Ok(vec![error]);
5403 }
5404 };
5405 timings.principal = principal_label(&principal);
5406
5407 let build_class = identity.build_class.take().filter(|build_class| {
5410 if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
5411 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
5412 } else {
5413 if self
5414 .build_class_warned
5415 .lock()
5416 .unwrap_or_else(|p| p.into_inner())
5417 .insert(ctx.connection_id)
5418 {
5419 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
5420 "ignoring build_class from a non-Callosum opener");
5421 }
5422 false
5423 }
5424 });
5425
5426 if let Principal::Reserved {
5430 module_id: opening_module_id,
5431 } = &principal
5432 {
5433 if let Some(opening_registration) = admission_registry
5434 .iter()
5435 .find(|r| r.manifest.module_id == *opening_module_id)
5436 {
5437 if let Some(capability) =
5438 denied_capability(&opening_registration.manifest, ®istration.manifest)
5439 {
5440 warn!(
5441 opening_module_id,
5442 target_module_id,
5443 capability,
5444 "refusing route.open because an attested capability deny edge matches"
5445 );
5446 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5447 &frame,
5448 &target_module_id,
5449 "capability_deny_edge",
5450 "capability_forbidden",
5451 format!(
5452 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
5453 ),
5454 )?]);
5455 }
5456 }
5457 }
5458
5459 if admission_facts.is_some() {
5460 let carrier_matches = matches!(
5461 &principal,
5462 Principal::Reserved { module_id }
5463 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
5464 );
5465 if !carrier_matches {
5466 return Ok(vec![self.route_open_refusal_frame(
5467 ctx,
5468 &caller,
5469 &frame,
5470 &target_module_id,
5471 "admission_facts_carrier_not_permitted",
5472 "admission_facts_not_permitted",
5473 "admission facts may only be carried by the configured reserved module",
5474 )?]);
5475 }
5476
5477 let target_allowed = self
5478 .admission_facts_targets
5479 .as_ref()
5480 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
5481 if !target_allowed {
5482 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5483 &frame,
5484 &target_module_id,
5485 "admission_facts_target_not_listed",
5486 "admission_facts_target_not_allowed",
5487 format!(
5488 "admission facts are not permitted for target module_id '{target_module_id}'"
5489 ),
5490 )?]);
5491 }
5492
5493 }
5497
5498 let (bound_scope, mut scope_stamp, stamp_required) = match &scope {
5504 None => (None, None, false),
5505 Some(selector) => {
5506 let owner_configured = match &selector.owner {
5507 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5513 _ => false,
5514 };
5515 let mut scopes = self
5516 .scopes
5517 .write()
5518 .unwrap_or_else(|poisoned| poisoned.into_inner());
5519 let admitted = crate::scopes::admission::admit_route(
5520 &scopes,
5521 &admission_registry,
5522 &principal,
5523 &target_module_id,
5524 Some(selector),
5525 owner_configured,
5526 crate::scopes::admission::AdmissionMode::Open,
5527 )
5528 .map(|route| {
5529 let admission = route.scope.expect("scoped admission");
5530 let bound = scopes.reserve_bind(
5531 &admission,
5532 &principal,
5533 ctx.connection_id,
5534 selector.hold_presence,
5535 );
5536 (admission, bound)
5537 });
5538 match admitted {
5539 Ok((admission, bound)) => {
5540 (Some(bound), Some(admission.stamp), admission.stamp_required)
5541 }
5542 Err(refusal) => {
5543 return Ok(vec![self.route_open_refusal_frame(
5544 ctx,
5545 &caller,
5546 &frame,
5547 &target_module_id,
5548 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
5549 refusal.code.unwrap_or(error_codes::TARGET_UNAVAILABLE),
5550 refusal.message,
5551 )?]);
5552 }
5553 }
5554 }
5555 };
5556 if let Some(stamp) = &scope_stamp {
5559 caller.scope = route_open_scope_label(stamp);
5560 }
5561
5562 timings.enter(RouteOpenPhase::ProjectRoot);
5570 let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
5571 Ok(project) => project,
5572 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5573 };
5574 timings.enter(RouteOpenPhase::Admission);
5575 let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
5576 Ok(root) => root,
5577 Err(err) => {
5578 return Ok(vec![control_error_frame(
5579 &frame,
5580 "invalid_project_root",
5581 err.to_string(),
5582 )?])
5583 }
5584 };
5585 let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
5591
5592 let _concurrency_guard = match self
5603 .route_bind_concurrency
5604 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5605 {
5606 Ok(guard) => guard,
5607 Err(in_flight) => {
5608 return Ok(vec![self.route_open_target_capacity_refusal(
5609 ctx,
5610 &caller,
5611 &frame,
5612 &target_module_id,
5613 in_flight,
5614 )?]);
5615 }
5616 };
5617
5618 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5621 RouteBindAdmission::Admitted { guard, probe } => {
5622 if probe {
5623 info!(
5624 module_id = %target_module_id,
5625 connection_id = ctx.connection_id.get(),
5626 "route.bind breaker half-open: admitting one probe"
5627 );
5628 }
5629 guard
5630 }
5631 RouteBindAdmission::Refused {
5632 consecutive_timeouts,
5633 retry_in,
5634 probe_in_flight,
5635 } => {
5636 return Ok(vec![self.route_open_breaker_refusal_frame(
5637 ctx,
5638 &caller,
5639 &frame,
5640 &target_module_id,
5641 consecutive_timeouts,
5642 retry_in,
5643 probe_in_flight,
5644 )?]);
5645 }
5646 };
5647
5648 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5653 let relay_deadline = Instant::now() + route_bind_relay_timeout;
5654 let operator_app = self
5655 .operator_apps
5656 .lock()
5657 .unwrap_or_else(|p| p.into_inner())
5658 .get(&ctx.connection_id)
5659 .cloned();
5660 timings.enter(RouteOpenPhase::EgressReservation);
5661 let pending = match self
5662 .forwarding
5663 .begin_route_bind_relay_for(
5664 ctx.connection_id,
5665 ctx.egress.clone(),
5666 response_version(&frame),
5667 frame.header.corr,
5668 &target_module_id,
5669 principal.clone(),
5670 bound_scope,
5671 project_root,
5672 identity.harness.clone(),
5673 build_class,
5674 operator_app.clone(),
5675 relay_deadline,
5676 busy_refusals == Some(true),
5677 )
5678 .await
5679 {
5680 Ok(pending) => pending,
5681 Err(err) => {
5682 return Ok(vec![self.route_open_refusal_frame(
5683 ctx,
5684 &caller,
5685 &frame,
5686 &target_module_id,
5687 "relay_reservation_failed",
5688 forwarding_error_code(&err),
5689 err.to_string(),
5690 )?])
5691 }
5692 };
5693 timings.enter(RouteOpenPhase::Admission);
5694 let crate::forwarding::PendingRouteBindRelay {
5695 endpoint,
5696 module_sink,
5697 negotiated_ver,
5698 client_channel,
5699 client_epoch,
5700 module_channel,
5701 module_epoch,
5702 corr: relay_corr,
5703 receiver,
5704 } = pending;
5705 let mut reservation =
5706 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5707
5708 let relay_registration = self
5713 .registry
5714 .get_module_by_connection(endpoint.connection_id)
5715 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5716 let relay_capabilities = relay_registration
5717 .as_ref()
5718 .and_then(|registration| registration.manifest.capabilities.as_ref());
5719 if let Some(selector) = &scope {
5720 if let Err(refusal) = self
5723 .scopes
5724 .read()
5725 .unwrap_or_else(|poisoned| poisoned.into_inner())
5726 .check_bind_proof(&principal, selector)
5727 {
5728 reservation.release_and_disarm();
5729 return Ok(vec![self.route_open_refusal_frame(
5730 ctx,
5731 &caller,
5732 &frame,
5733 &target_module_id,
5734 refusal.code,
5735 refusal.code,
5736 refusal.message,
5737 )?]);
5738 }
5739 }
5740 if let Some(stamp) = scope_stamp.as_ref() {
5741 if let Err(refusal) = crate::scopes::check_target_required_stamp(
5742 stamp_required,
5743 &target_module_id,
5744 relay_capabilities,
5745 )
5746 .and_then(|()| {
5747 crate::scopes::check_target_flow_support(
5748 stamp,
5749 &target_module_id,
5750 relay_capabilities,
5751 )
5752 })
5753 .and_then(|()| {
5754 crate::scopes::check_target_agent_run_support(
5755 stamp,
5756 &target_module_id,
5757 relay_capabilities,
5758 )
5759 }) {
5760 reservation.release_and_disarm();
5761 return Ok(vec![self.route_open_refusal_frame(
5762 ctx,
5763 &caller,
5764 &frame,
5765 &target_module_id,
5766 refusal.code,
5767 refusal.code,
5768 refusal.message,
5769 )?]);
5770 }
5771 }
5772
5773 if scope_stamp
5776 .as_ref()
5777 .is_some_and(|stamp| stamp.kind == subc_protocol::scope::ScopeKind::Federated)
5778 && !crate::scopes::target_supports_federated(relay_capabilities)
5779 {
5780 scope_stamp = None;
5782 }
5783 if !relay_capabilities.is_some_and(|caps| {
5784 caps.provides
5785 .iter()
5786 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5787 }) {
5788 if let Some(stamp) = scope_stamp.as_mut() {
5789 stamp.workdir = None;
5790 stamp.project_id = None;
5791 }
5792 }
5793 let worker_run = scope_stamp
5798 .as_ref()
5799 .is_some_and(crate::scopes::is_worker_run)
5800 && relay_capabilities.is_some_and(|caps| {
5801 caps.provides
5802 .iter()
5803 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5804 });
5805 if !worker_run
5806 && !relay_capabilities.is_some_and(|caps| {
5807 caps.provides
5808 .iter()
5809 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5810 })
5811 {
5812 if let Some(stamp) = scope_stamp.as_mut() {
5813 stamp.attributes.on_behalf_of_agent = None;
5814 }
5815 }
5816 if !relay_capabilities.is_some_and(|caps| {
5817 caps.provides
5818 .iter()
5819 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5820 }) {
5821 if let Some(stamp) = scope_stamp.as_mut() {
5822 stamp.attributes.flow_version = None;
5823 stamp.attributes.flow_code_hash = None;
5824 stamp.attributes.flow_install_id = None;
5825 }
5826 }
5827 let call_mode = if worker_run
5828 || relay_capabilities
5829 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5830 {
5831 Some(derive_call_mode(scope_stamp.as_ref()))
5832 } else {
5833 if let Some(stamp) = scope_stamp.as_mut() {
5834 stamp.expires_at_ms = None;
5835 }
5836 None
5837 };
5838
5839 debug!(
5840 connection_id = ctx.connection_id.get(),
5841 client_channel,
5842 client_epoch,
5843 module_channel,
5844 module_epoch,
5845 "reserved route handle pair"
5846 );
5847 let route_principal_label = principal_label(&principal);
5850 let relay = ModuleControlRequest::RouteBind {
5851 route_channel: module_channel,
5852 epoch: module_epoch,
5853 target,
5854 identity,
5855 principal: Some(principal),
5856 consumer_capabilities,
5857 role_versions,
5858 admission_facts,
5859 scope: scope_stamp,
5860 call_mode,
5861 };
5862 let peer = RouteBindPeer::new()
5863 .with_observed_peer(
5864 relay_capabilities
5865 .is_some_and(|caps| {
5866 caps.provides
5867 .iter()
5868 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5869 })
5870 .then(|| {
5871 self.observed_peers
5872 .lock()
5873 .unwrap_or_else(|p| p.into_inner())
5874 .get(&ctx.connection_id)
5875 .cloned()
5876 .unwrap_or_else(ObservedPeer::unobserved)
5877 }),
5878 )
5879 .with_operator_app(
5880 self.connection_operator_app(operator_app.as_ref())
5881 .filter(|_| {
5882 relay_capabilities.is_some_and(|caps| {
5883 caps.provides
5884 .iter()
5885 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5886 })
5887 }),
5888 );
5889 let relay_body = crate::bind_compat::encode_bind(
5890 &relay,
5891 &peer,
5892 &legacy_bind_project,
5893 relay_capabilities,
5894 )
5895 .map_err(|err| {
5896 RouterError::backend(
5897 0,
5898 frame.header.corr,
5899 format!("failed to encode route.bind request: {err}"),
5900 )
5901 })?;
5902 let relay_frame = Frame::build_with_version(
5903 negotiated_ver,
5904 FrameType::Request,
5905 control_flags(),
5906 0,
5907 0,
5908 relay_corr,
5909 relay_body,
5910 )
5911 .map_err(RouterError::FrameBuild)?;
5912
5913 timings.enter(RouteOpenPhase::BindSend);
5914 let sent = module_sink.send(relay_frame).await;
5915 timings.enter(RouteOpenPhase::Admission);
5916 if let Err(err) = sent {
5917 reservation.release_and_disarm();
5918 return Ok(vec![self.route_open_refusal_frame(
5919 ctx,
5920 &caller,
5921 &frame,
5922 &target_module_id,
5923 "relay_send_failed",
5924 "target_unavailable",
5925 err.to_string(),
5926 )?]);
5927 }
5928
5929 #[cfg(test)]
5930 if let Some(hold) = &self.route_bind_relay_sent_hold {
5931 hold.wait().await;
5935 hold.wait().await;
5936 }
5937
5938 if self
5943 .forwarding
5944 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5945 .map_err(RouterError::Forwarding)?
5946 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5947 {
5948 self.send_abandoned_route_bind_goodbye(
5949 &module_sink,
5950 negotiated_ver,
5951 module_channel,
5952 module_epoch,
5953 );
5954 }
5955
5956 timings.enter(RouteOpenPhase::BindAckWait);
5957 let outcome = timeout_at(relay_deadline, receiver).await;
5958 timings.enter(RouteOpenPhase::Admission);
5959 match outcome {
5960 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5961 reservation.disarm();
5962 if breaker.record_accepted() {
5963 info!(
5964 module_id = %target_module_id,
5965 "route.bind breaker closed: the probe was accepted"
5966 );
5967 }
5968 self.observe_route_open_accept(
5969 ctx,
5970 &target_module_id,
5971 &route_principal_label,
5972 &caller.scope,
5973 );
5974 Ok(Vec::new())
5975 }
5976 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5977 reservation.release_and_disarm();
5978 breaker.record_inconclusive();
5982 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5983 }
5984 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5985 reservation.release_and_disarm();
5986 breaker.record_inconclusive();
5987 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5988 }
5989 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5990 reservation.release_and_disarm();
5991 breaker.record_inconclusive();
5992 tracing::warn!(
5999 module_id = %target_module_id,
6000 "route.bind relay abandoned: {message}"
6001 );
6002 Ok(vec![self.route_open_refusal_frame(
6003 ctx,
6004 &caller,
6005 &frame,
6006 &target_module_id,
6007 "relay_abandoned",
6008 "target_unavailable",
6009 message,
6010 )?])
6011 }
6012 Ok(Err(_)) => {
6013 reservation.release_and_disarm();
6014 breaker.record_inconclusive();
6015 Ok(vec![self.route_open_refusal_frame(
6016 ctx,
6017 &caller,
6018 &frame,
6019 &target_module_id,
6020 "relay_waiter_canceled",
6021 "target_unavailable",
6022 "route.bind relay waiter was canceled before the module responded",
6023 )?])
6024 }
6025 Err(_) => {
6026 reservation.release_and_disarm();
6027 if let Some(opened) = breaker.record_timeout(
6031 self.route_bind_breaker_threshold,
6032 self.route_bind_breaker_cooldown,
6033 ) {
6034 warn!(
6035 module_id = %target_module_id,
6036 consecutive_timeouts = opened.consecutive_timeouts,
6037 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
6038 reopened_after_probe = opened.reopened_after_probe,
6039 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
6040 );
6041 }
6042 tracing::warn!(
6050 module_id = %target_module_id,
6051 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
6052 "route.bind relay timed out: module did not ack within budget"
6053 );
6054 Ok(vec![self.route_open_refusal_frame(
6055 ctx,
6056 &caller,
6057 &frame,
6058 &target_module_id,
6059 "relay_timed_out",
6060 "module_timeout",
6061 format!(
6062 "module_id '{target_module_id}' did not answer route.bind within {:?}",
6063 route_bind_relay_timeout
6064 ),
6065 )?])
6066 }
6067 }
6068 }
6069
6070 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6071 let response = ClientControlResponse::SupervisorSpawnSnapshot {
6072 snapshot: self.supervisor.spawn_snapshot(),
6073 };
6074 Ok(vec![control_response_body_frame(
6075 &frame,
6076 &response,
6077 "ClientControlResponse::SupervisorSpawnSnapshot",
6078 )?])
6079 }
6080
6081 fn handle_supervisor_spawn_subscribe(
6082 &self,
6083 ctx: &RouteCtx,
6084 frame: Frame,
6085 since: Option<SpawnCursor>,
6086 ) -> Result<Vec<Frame>, RouterError> {
6087 match self.supervisor.subscribe_spawns(
6088 ctx.connection_id,
6089 frame.header.corr,
6090 response_version(&frame),
6091 since,
6092 ctx.egress.clone(),
6093 ) {
6094 Ok(()) => Ok(Vec::new()),
6095 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
6096 Ok(vec![control_error_body_frame(
6097 &frame,
6098 ErrorBody {
6099 code: "spawn_cursor_incarnation_mismatch".to_string(),
6100 message: "spawn cursor belongs to a different daemon incarnation"
6101 .to_string(),
6102 detail: Some(serde_json::json!({
6103 "current_daemon_incarnation": current
6104 })),
6105 },
6106 )?])
6107 }
6108 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
6109 &frame,
6110 ErrorBody {
6111 code: "spawn_cursor_too_old".to_string(),
6112 message: "spawn cursor predates the retained event ring".to_string(),
6113 detail: Some(serde_json::json!({
6114 "oldest_retained_cursor": oldest
6115 })),
6116 },
6117 )?]),
6118 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
6119 0,
6120 frame.header.corr,
6121 format!("failed to open supervisor spawn subscription: {error}"),
6122 )),
6123 }
6124 }
6125
6126 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6127 let handler = self.clone();
6130 let runtime = tokio::runtime::Handle::current();
6131 let subscriber = tracing::dispatcher::get_default(Clone::clone);
6134 let corr = frame.header.corr;
6135 tokio::task::spawn_blocking(move || {
6136 runtime.block_on(
6137 list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
6138 .with_subscriber(subscriber),
6139 )
6140 })
6141 .await
6142 .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
6143 }
6144
6145 async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6146 let generation = self
6147 .registry
6148 .generation()
6149 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6150 let supervised = self.supervisor.list();
6151 let mut statuses = BTreeMap::new();
6152 let mut not_running = BTreeSet::new();
6153 for module in &supervised {
6154 let status = module.status_for_control("list").map_err(|err| {
6155 RouterError::backend(
6156 0,
6157 frame.header.corr,
6158 format!("failed to read supervisor status: {err}"),
6159 )
6160 })?;
6161 if !status.enabled || !status.process_alive {
6162 not_running.insert(module.module_id().to_owned());
6163 }
6164 statuses.insert(module.module_id().to_owned(), status);
6165 }
6166 let mut process_resources = BTreeMap::new();
6169 for module in &supervised {
6170 process_resources.insert(
6171 module.module_id().to_owned(),
6172 crate::subtree_resources::read_process(module.clone()).await,
6173 );
6174 }
6175 let mut image_readings = BTreeMap::new();
6178 for module in &supervised {
6179 image_readings.insert(
6180 module.module_id().to_owned(),
6181 module.running_image_agreement().await,
6182 );
6183 }
6184 let readings = self.resource_tracker.list(supervised.clone());
6185 let mut modules = Vec::new();
6186 for module in supervised {
6187 let status = statuses
6188 .remove(module.module_id())
6189 .expect("each supervised module has a status snapshot");
6190 let (configured, _) = module.configuration().map_err(|err| {
6191 RouterError::backend(
6192 0,
6193 frame.header.corr,
6194 format!("failed to read module configuration: {err}"),
6195 )
6196 })?;
6197 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
6198 RouterError::backend(
6199 0,
6200 frame.header.corr,
6201 format!("failed to read retry time: {err}"),
6202 )
6203 })?;
6204 let image = image_readings
6205 .remove(module.module_id())
6206 .expect("each supervised module has an image snapshot");
6207 let resources = process_resources.remove(module.module_id());
6208 let mut verdict =
6209 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
6210 verdict.spec = Some(launch_spec_agreement(
6211 &configured,
6212 status.process_alive,
6213 status.launch_spec_digest.as_ref(),
6214 ));
6215 let pending_reload = Some(verdict);
6216 let configured_enabled = self
6217 .capability_evaluator
6218 .configured_enabled(&status.module_id);
6219 modules.push(
6222 SupervisorEntry::new(
6223 status.module_id,
6224 status.state.to_string(),
6225 status.enabled,
6226 status.live,
6227 status.health.status,
6228 )
6229 .with_configured_enabled(configured_enabled)
6230 .with_next_retry_at_ms(next_retry_at_ms)
6231 .with_launch_nonce_env(Some(!cfg!(unix)))
6232 .with_launch_nonce_source(status.launch_nonce_source)
6233 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
6234 .with_protocol(status.protocol)
6235 .with_pending_reload(pending_reload)
6236 .with_last_probe_ms(status.health.last_probe_ms)
6237 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
6238 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
6239 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
6240 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
6241 .with_restart_count(Some(status.restart_count))
6242 .with_max_restarts(Some(status.max_restarts))
6243 .with_lifetime_restarts(Some(status.lifetime_restarts))
6244 .with_spawn_generation(Some(status.spawn_generation))
6245 .with_restart_window_secs(Some(status.restart_window.as_secs()))
6246 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
6247 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
6248 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
6249 .with_resources(resources),
6250 );
6251 }
6252 for module in &mut modules {
6253 if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
6254 module.tree = tree.clone();
6255 module.cpu_last_hour = hour.clone();
6256 }
6257 module.tree_unavailable = readings
6258 .reasons
6259 .get(&module.module_id)
6260 .map(|reason| (*reason).to_owned());
6261 if module.tree.is_none() {
6262 module.tree_unavailable = Some(
6263 if not_running.contains(&module.module_id) {
6264 "not_running"
6265 } else if let Some(reason) = readings.reasons.get(&module.module_id) {
6266 reason
6267 } else {
6268 "not_yet_measured"
6269 }
6270 .to_owned(),
6271 );
6272 }
6273 }
6274 let response = ClientControlResponse::SupervisorList {
6275 generation,
6276 modules,
6277 daemon: readings.daemon,
6278 agent_commands: readings.agent_commands,
6279 };
6280 Ok(vec![control_response_body_frame(
6281 &frame,
6282 &response,
6283 "ClientControlResponse::SupervisorList",
6284 )?])
6285 }
6286
6287 fn handle_supervisor_stderr_tail(
6288 &self,
6289 frame: Frame,
6290 module_id: String,
6291 max_lines: Option<u32>,
6292 max_bytes: Option<u32>,
6293 ) -> Result<Vec<Frame>, RouterError> {
6294 let Some(module) = self.supervisor.get(&module_id) else {
6295 return Ok(vec![control_error_frame(
6296 &frame,
6297 "unknown_module",
6298 format!("module_id '{module_id}' is not supervised"),
6299 )?]);
6300 };
6301
6302 let snapshot = module.stderr_tail(
6303 max_lines.map(|value| value as usize),
6304 max_bytes.map(|value| value as usize),
6305 );
6306
6307 let response = ClientControlResponse::SupervisorStderrTail {
6308 module_id,
6309 tail: StderrTail {
6310 capture: match snapshot.capture {
6311 CaptureState::Captured => StderrCaptureState::Captured,
6312 CaptureState::Incomplete { reason } => {
6313 StderrCaptureState::Incomplete { reason }
6314 }
6315 CaptureState::NotCaptured { reason } => {
6316 StderrCaptureState::NotCaptured { reason }
6317 }
6318 },
6319 entries: snapshot
6320 .entries
6321 .into_iter()
6322 .map(|entry| match entry {
6323 TailEntry::Line {
6324 text,
6325 truncated,
6326 at_ms,
6327 } => StderrTailEntry::Line {
6328 text,
6329 truncated,
6330 at_ms,
6331 },
6332 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
6333 })
6334 .collect(),
6335 dropped_lines: snapshot.dropped_lines,
6336 },
6337 };
6338 Ok(vec![control_response_body_frame(
6339 &frame,
6340 &response,
6341 "ClientControlResponse::SupervisorStderrTail",
6342 )?])
6343 }
6344
6345 async fn observe_control_wait<F: std::future::Future>(
6346 &self,
6347 ctx: &RouteCtx,
6348 target: Option<&str>,
6349 kind: &'static str,
6350 future: F,
6351 ) -> F::Output {
6352 self.counters
6353 .observe_reader_wait(
6354 || {
6355 let module = self
6356 .forwarding
6357 .module_id_for_connection(ctx.connection_id)
6358 .ok()
6359 .flatten();
6360 let principal = if module.is_some() { "module" } else { "direct" };
6361 (
6362 self.resolve_reader_wait_target(kind, target)
6363 .or(module)
6364 .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
6365 principal.into(),
6366 )
6367 },
6368 kind,
6369 future,
6370 )
6371 .await
6372 }
6373
6374 pub(crate) fn resolve_reader_wait_target(
6377 &self,
6378 op: &str,
6379 selector: Option<&str>,
6380 ) -> Option<String> {
6381 let selector = selector?;
6382 match op {
6383 ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
6384 .supervisor
6385 .get(selector)
6386 .map(|module| module.module_id().to_owned()),
6387 ops::SUPERVISOR_HEALTH_PROBE => self
6388 .registry
6389 .get_module(selector)
6390 .ok()
6391 .flatten()
6392 .map(|registration| registration.manifest.module_id),
6393 _ => None,
6394 }
6395 }
6396
6397 async fn handle_supervisor_terminals(
6398 &self,
6399 ctx: &RouteCtx,
6400 frame: Frame,
6401 module_id: String,
6402 ) -> Result<Vec<Frame>, RouterError> {
6403 let Some(module) = self.supervisor.get(&module_id) else {
6404 return Ok(vec![control_error_frame(
6405 &frame,
6406 "unknown_module",
6407 format!("module_id '{module_id}' is not supervised"),
6408 )?]);
6409 };
6410
6411 let terminals = self
6414 .observe_control_wait(
6415 ctx,
6416 Some(module.module_id()),
6417 ops::SUPERVISOR_TERMINALS,
6418 module.read_durable_terminal_history(),
6419 )
6420 .await
6421 .map_err(|error| {
6422 RouterError::backend(
6423 0,
6424 frame.header.corr,
6425 format!("failed to read terminal history: {error}"),
6426 )
6427 })?;
6428 let response = ClientControlResponse::SupervisorTerminals {
6429 module_id,
6430 terminals,
6431 };
6432 Ok(vec![control_response_body_frame(
6433 &frame,
6434 &response,
6435 "ClientControlResponse::SupervisorTerminals",
6436 )?])
6437 }
6438
6439 fn handle_supervisor_routes(
6440 &self,
6441 frame: Frame,
6442 module_id: Option<String>,
6443 ) -> Result<Vec<Frame>, RouterError> {
6444 let modules = self
6445 .forwarding
6446 .route_census(module_id.as_deref())
6447 .map_err(RouterError::Forwarding)?
6448 .into_iter()
6449 .map(|(module_id, routes)| SupervisorRouteModule {
6450 module_id,
6451 routes: routes
6452 .into_iter()
6453 .map(|route| SupervisorRoute {
6454 consumer: match route.principal {
6455 Principal::Reserved { module_id } => {
6456 SupervisorRouteConsumer::Reserved { module_id }
6457 }
6458 Principal::Direct | Principal::Unverified => {
6459 SupervisorRouteConsumer::Direct {
6460 connection_id: route.goodbye_target.connection_id.get(),
6461 }
6462 }
6463 },
6464 age_ms: Instant::now()
6465 .saturating_duration_since(route.bound_at)
6466 .as_millis()
6467 .try_into()
6468 .unwrap_or(u64::MAX),
6469 draining: route.draining,
6470 drain_reason: route.drain_reason,
6471 })
6472 .collect(),
6473 })
6474 .collect();
6475 let response = ClientControlResponse::SupervisorRoutes { modules };
6476 Ok(vec![control_response_body_frame(
6477 &frame,
6478 &response,
6479 "ClientControlResponse::SupervisorRoutes",
6480 )?])
6481 }
6482
6483 async fn handle_supervisor_provenance(
6484 &self,
6485 ctx: &RouteCtx,
6486 frame: Frame,
6487 module_id: Option<String>,
6488 ) -> Result<Vec<Frame>, RouterError> {
6489 let targeted = module_id.is_some();
6490 let selected = if let Some(module_id) = module_id {
6491 let Some(module) = self.supervisor.get(&module_id) else {
6492 return Ok(vec![control_error_frame(
6493 &frame,
6494 "unknown_module",
6495 format!("module_id '{module_id}' is not supervised"),
6496 )?]);
6497 };
6498 vec![module]
6499 } else {
6500 self.supervisor.list()
6501 };
6502
6503 let target = targeted.then(|| selected[0].module_id());
6504 self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
6505 let mut modules = Vec::with_capacity(selected.len());
6506 for module in &selected {
6507 let (status, observed_image) = module
6508 .status_and_running_image_agreement()
6509 .await
6510 .map_err(|err| {
6511 RouterError::backend(
6512 0,
6513 frame.header.corr,
6514 format!("failed to read supervisor status: {err}"),
6515 )
6516 })?;
6517 let module_declared = self
6518 .registry
6519 .get_module(&status.module_id)
6520 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6521 .and_then(|registration| registration.manifest.provenance)
6522 .map(|build| ModuleDeclaredProvenance::Reported { build })
6523 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
6524 #[cfg(test)]
6525 let running_image = match &self.provenance_probe_override {
6526 Some(result) => result.clone(),
6527 None => observed_image,
6528 };
6529 #[cfg(not(test))]
6530 let running_image = observed_image;
6531 modules.push(SupervisorModuleProvenance {
6532 module_id: status.module_id,
6533 module_declared,
6534 daemon_observed: SupervisorObservedProcess {
6535 launch_nonce_source: status.launch_nonce_source,
6536 pid: status.pid,
6537 spawned_at_ms: status.spawned_at_ms,
6538 spawned_from: status.spawned_from,
6539 running_image,
6540 },
6541 });
6542 }
6543 let daemon = SupervisorDaemonProvenance {
6544 daemon_build: self.daemon_provenance.build.clone(),
6545 daemon_observed: DaemonObservedProcess {
6546 pid: self.daemon_provenance.pid,
6547 started_at_ms: self
6548 .daemon_provenance
6549 .start_clock
6550 .map(|clock| clock.started_at_ms())
6551 .or(self.daemon_provenance.started_at_ms),
6552 running_image: self
6553 .daemon_provenance
6554 .probe
6555 .observe(
6556 self.daemon_provenance.pid,
6557 self.daemon_provenance.executable_path.as_deref(),
6558 self.daemon_provenance.executable_identity.clone(),
6559 self.daemon_provenance.process_start_time,
6560 )
6561 .await,
6562 },
6563 };
6564 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
6565 Ok(vec![control_response_body_frame(
6566 &frame,
6567 &response,
6568 "ClientControlResponse::SupervisorProvenance",
6569 )?])
6570 })
6571 .await
6572 }
6573
6574 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6575 self.refresh_capability_requirements();
6576 let generation = self
6577 .registry
6578 .generation()
6579 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6580 let modules = self
6581 .supervisor
6582 .list()
6583 .into_iter()
6584 .map(|module| {
6585 let status = module.status_for_control("health").map_err(|err| {
6586 RouterError::backend(
6587 0,
6588 frame.header.corr,
6589 format!("failed to read supervisor health: {err}"),
6590 )
6591 })?;
6592 let module_id = status.module_id;
6593 let capability_detail = self
6594 .capability_evaluator
6595 .required_problem_detail(&module_id);
6596 Ok(SupervisorHealthEntry {
6597 module_id,
6598 status: status.health.status,
6599 detail: append_capability_problem_detail(
6600 status.health.detail,
6601 capability_detail,
6602 ),
6603 metrics: status.health.metrics,
6604 consecutive_failures: status.health.consecutive_failures,
6605 late_answer_count: status.health.late_answer_count,
6606 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
6607 last_action: status.health.last_action,
6608 last_action_ms: status.health.last_action_ms,
6609 last_probe_ms: status.health.last_probe_ms,
6610 })
6611 })
6612 .collect::<Result<Vec<_>, RouterError>>()?;
6613 let response = ClientControlResponse::SupervisorHealth {
6614 generation,
6615 modules,
6616 };
6617 Ok(vec![control_response_body_frame(
6618 &frame,
6619 &response,
6620 "ClientControlResponse::SupervisorHealth",
6621 )?])
6622 }
6623
6624 async fn handle_supervisor_restart(
6625 &self,
6626 frame: Frame,
6627 module_id: String,
6628 drain_timeout_ms: Option<u64>,
6629 requester: Principal,
6630 ) -> Result<Vec<Frame>, RouterError> {
6631 let operation_lock = self.supervisor.operation_lock();
6632 let _operation_guard = operation_lock.lock().await;
6633 let Some(module) = self.supervisor.get(&module_id) else {
6634 return Ok(vec![control_error_frame(
6635 &frame,
6636 "unknown_module",
6637 format!("module_id '{module_id}' is not supervised"),
6638 )?]);
6639 };
6640
6641 self.route_outages.mark_operator_action(&module_id);
6642 if let Err(err) = module
6643 .restart_requested(drain_timeout_ms, Some(requester))
6644 .await
6645 {
6646 self.route_outages
6647 .operator_action_ended_unrefused(&module_id);
6648 let (code, message) = match err {
6649 crate::supervise::SuperviseError::Disabled { .. } => {
6650 ("module_disabled", err.to_string())
6651 }
6652 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6653 ("swap_in_progress", err.to_string())
6654 }
6655 _ => (
6656 "target_unavailable",
6657 format!("failed to restart module_id '{module_id}': {err}"),
6658 ),
6659 };
6660 return Ok(vec![control_error_frame(&frame, code, message)?]);
6661 }
6662
6663 let response = ClientControlResponse::SupervisorAck {
6664 module_id,
6665 applied: true,
6666 };
6667 Ok(vec![control_response_body_frame(
6668 &frame,
6669 &response,
6670 "ClientControlResponse::SupervisorAck",
6671 )?])
6672 }
6673
6674 async fn handle_supervisor_swap(
6678 &self,
6679 frame: Frame,
6680 module_id: String,
6681 ready_timeout_ms: Option<u64>,
6682 requester: Principal,
6683 ) -> Result<Vec<Frame>, RouterError> {
6684 let module = {
6691 let operation_lock = self.supervisor.operation_lock();
6692 let _operation_guard = operation_lock.lock().await;
6693 self.supervisor.get(&module_id)
6694 };
6695 let Some(module) = module else {
6696 return Ok(vec![control_error_frame(
6697 &frame,
6698 "unknown_module",
6699 format!("module_id '{module_id}' is not supervised"),
6700 )?]);
6701 };
6702
6703 self.route_outages.mark_operator_action(&module_id);
6704 if let Err(err) = module
6705 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6706 .await
6707 {
6708 self.route_outages
6709 .operator_action_ended_unrefused(&module_id);
6710 use crate::supervise::SuperviseError;
6711 let message = err.to_string();
6712 let error = match err {
6713 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6714 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6715 code: "swap_refused".to_string(),
6716 message,
6717 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6718 },
6719 SuperviseError::SwapFailed {
6720 arm,
6721 candidate_exit,
6722 ..
6723 } => ErrorBody {
6724 code: "swap_failed".to_string(),
6725 message,
6726 detail: Some(serde_json::json!({
6727 "arm": arm.as_str(),
6728 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6729 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6730 })),
6731 },
6732 _ => ErrorBody::new(
6733 "target_unavailable",
6734 format!("failed to swap module_id '{module_id}': {message}"),
6735 ),
6736 };
6737 return Ok(vec![control_error_body_frame(&frame, error)?]);
6738 }
6739 self.route_outages
6743 .operator_action_ended_unrefused(&module_id);
6744
6745 let response = ClientControlResponse::SupervisorAck {
6746 module_id,
6747 applied: true,
6748 };
6749 Ok(vec![control_response_body_frame(
6750 &frame,
6751 &response,
6752 "ClientControlResponse::SupervisorAck",
6753 )?])
6754 }
6755
6756 async fn handle_supervisor_reload(
6757 &self,
6758 frame: Frame,
6759 module_id: String,
6760 requester: Principal,
6761 ) -> Result<Vec<Frame>, RouterError> {
6762 let operation_lock = self.supervisor.operation_lock();
6763 let _operation_guard = operation_lock.lock().await;
6764 let Some(module) = self.supervisor.get(&module_id) else {
6765 return Ok(vec![control_error_frame(
6766 &frame,
6767 "unknown_module",
6768 format!("module_id '{module_id}' is not supervised"),
6769 )?]);
6770 };
6771
6772 self.route_outages.mark_operator_action(&module_id);
6773 if let Err(err) = module.reload_requested(Some(requester)).await {
6774 self.route_outages
6775 .operator_action_ended_unrefused(&module_id);
6776 let (code, message) = match err {
6777 crate::supervise::SuperviseError::Disabled { .. } => {
6778 ("module_disabled", err.to_string())
6779 }
6780 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6781 ("swap_in_progress", err.to_string())
6782 }
6783 _ => (
6784 "reload_failed",
6785 format!("failed to reload module_id '{module_id}': {err}"),
6786 ),
6787 };
6788 return Ok(vec![control_error_frame(&frame, code, message)?]);
6789 }
6790
6791 let response = ClientControlResponse::SupervisorAck {
6792 module_id,
6793 applied: true,
6794 };
6795 Ok(vec![control_response_body_frame(
6796 &frame,
6797 &response,
6798 "ClientControlResponse::SupervisorAck",
6799 )?])
6800 }
6801
6802 async fn handle_supervisor_rescan(
6806 &self,
6807 frame: Frame,
6808 mode: RescanMode,
6809 requester: Principal,
6810 ) -> Result<Vec<Frame>, RouterError> {
6811 let preview = mode.previews();
6812 let Some(context) = self.rescan.clone() else {
6813 return Ok(vec![control_error_frame(
6814 &frame,
6815 "rescan_unavailable",
6816 "the daemon was not started with a reloadable config path".to_string(),
6817 )?]);
6818 };
6819
6820 let operation_lock = self.supervisor.operation_lock();
6821 let _operation_guard = operation_lock.lock().await;
6822 let loaded = match self.load_rescan_config(&mode, &context) {
6826 Ok(config) => config,
6827 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6828 };
6829 let Some((config, activity_config)) = loaded else {
6837 return Ok(vec![control_error_frame(
6838 &frame,
6839 "invalid_daemon_config",
6840 format!(
6841 "daemon config not found at {}; refusing to rescan (an absent config would \
6842 retire every supervised module)",
6843 context.config_path.display()
6844 ),
6845 )?]);
6846 };
6847 let (
6848 configured_port,
6849 storage_config,
6850 admission_facts_carrier_module_id,
6851 admission_facts_targets,
6852 scope_authority_owners,
6853 federated_scope_owner,
6854 privileged_operator_prompt,
6855 modules,
6856 reserved_capabilities,
6857 ) = (
6858 config.port,
6859 config.storage,
6860 config.admission_facts_carrier_module_id,
6861 config.admission_facts_targets,
6862 config.scope_authority_owners,
6863 config.federated_scope_owner,
6864 config.privileged_operator_prompt,
6865 config.modules,
6866 config.reserved_capabilities,
6867 );
6868
6869 let mut restart_required = Vec::new();
6878 for section in RestartRequiredSection::ALL {
6879 let changed = match section {
6880 RestartRequiredSection::Port => configured_port != context.configured_port,
6881 RestartRequiredSection::Storage => storage_config != context.storage_config,
6882 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6883 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6884 }
6885 RestartRequiredSection::AdmissionFactsTargets => {
6886 admission_facts_targets != context.admission_facts_targets
6887 }
6888 RestartRequiredSection::ScopeAuthorityOwners => {
6889 scope_authority_owners != context.scope_authority_owners
6890 }
6891 RestartRequiredSection::FederatedScopeOwner => {
6892 federated_scope_owner != context.federated_scope_owner
6893 }
6894 RestartRequiredSection::PrivilegedOperatorPrompt => {
6895 privileged_operator_prompt != context.privileged_operator_prompt
6896 }
6897 };
6898 if changed {
6899 restart_required.push(section.label().to_string());
6900 }
6901 }
6902 if !restart_required.is_empty() {
6903 warn!(
6904 config_path = %context.config_path.display(),
6905 sections = %restart_required.join(", "),
6906 "daemon config changed outside the modules section; restart the daemon to apply those changes"
6907 );
6908 }
6909
6910 for configured in &modules {
6911 if let Err(err) = validate_spec(&configured.module_spec()) {
6912 return Ok(vec![control_error_frame(
6913 &frame,
6914 "invalid_daemon_config",
6915 format!("supervisor rescan rejected daemon config: {err}"),
6916 )?]);
6917 }
6918 }
6919
6920 let configured_capabilities = modules
6921 .iter()
6922 .map(|module| (module.module_id.clone(), module.enabled))
6923 .collect::<Vec<_>>();
6924 let preview_capability_warnings = if preview {
6925 let (_, registrations) = self.runtime_capability_snapshot()?;
6926 let current_modules = self
6927 .supervisor
6928 .list()
6929 .into_iter()
6930 .map(|module| module.module_id().to_string())
6931 .collect::<BTreeSet<_>>();
6932 let resulting_modules = configured_capabilities.clone();
6933 let removed = current_modules
6934 .into_iter()
6935 .filter(|module_id| {
6936 !resulting_modules
6937 .iter()
6938 .any(|(configured_id, _)| configured_id == module_id)
6939 })
6940 .collect::<Vec<_>>();
6941 let disabled = match &mode {
6944 RescanMode::Candidate(_) => resulting_modules
6945 .iter()
6946 .filter(|(_, enabled)| !enabled)
6947 .map(|(module_id, _)| module_id.clone())
6948 .collect::<Vec<_>>(),
6949 _ => Vec::new(),
6950 };
6951 self.capability_evaluator.preview_warnings(
6952 resulting_modules,
6953 &removed,
6954 &disabled,
6955 ®istrations,
6956 )
6957 } else {
6958 Vec::new()
6959 };
6960 let reconciled = match &mode {
6961 RescanMode::Candidate(candidate) => self
6962 .preview_candidate(&context, modules, candidate)
6963 .map(|result| (result, Vec::new())),
6964 _ => {
6965 self.reconcile_supervised_modules(
6966 &context.supervisor,
6967 modules,
6968 preview,
6969 requester.clone(),
6970 )
6971 .await
6972 }
6973 };
6974 let (result, changed) = match reconciled {
6975 Ok(result) => result,
6976 Err(message) => {
6977 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6978 }
6979 };
6980 if !preview {
6981 if let Some(ledger) = &self.activity {
6982 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6983 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6984 }
6985 if let Err(code) = self.configure_activity(activity_config).await {
6986 warn!(
6987 code,
6988 "activity retention could not be applied during rescan"
6989 );
6990 }
6991 self.capability_evaluator
6992 .configure(configured_capabilities, reserved_capabilities);
6993 self.capability_evaluator.wake_deadline_loop();
6994 self.refresh_capability_requirements();
6995 }
6996 let mut result = result;
6997 result.restart_required = restart_required;
6998 result.capability_warnings = preview_capability_warnings;
6999 let response = ClientControlResponse::SupervisorRescan { result };
7000 Ok(vec![control_response_body_frame(
7001 &frame,
7002 &response,
7003 "ClientControlResponse::SupervisorRescan",
7004 )?])
7005 }
7006
7007 async fn handle_supervisor_release_reserved(
7008 &self,
7009 frame: Frame,
7010 module_id: String,
7011 ) -> Result<Vec<Frame>, RouterError> {
7012 let Some(context) = self.rescan.clone() else {
7013 return Ok(vec![control_error_frame(
7014 &frame,
7015 "release_unavailable",
7016 "reserved-id release requires a daemon started with a reloadable config path",
7017 )?]);
7018 };
7019 let operation_lock = self.supervisor.operation_lock();
7020 let _operation_guard = operation_lock.lock().await;
7021 let loaded = match crate::daemon_config::load(&context.config_path) {
7022 Ok(Some(config)) => config,
7023 Ok(None) => {
7024 return Ok(vec![control_error_frame(
7025 &frame,
7026 "invalid_daemon_config",
7027 format!(
7028 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
7029 context.config_path.display()
7030 ),
7031 )?])
7032 }
7033 Err(err) => {
7034 return Ok(vec![control_error_frame(
7035 &frame,
7036 "invalid_daemon_config",
7037 format!("unable to verify reserved-id release against daemon config: {err}"),
7038 )?])
7039 }
7040 };
7041 if loaded
7042 .modules
7043 .iter()
7044 .any(|configured| configured.module_id == module_id)
7045 {
7046 return Ok(vec![control_error_frame(
7047 &frame,
7048 "reserved_module_configured",
7049 format!(
7050 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
7051 ),
7052 )?]);
7053 }
7054 if !self.supervisor.release_retained_reserved_gate(&module_id) {
7055 return Ok(vec![control_error_frame(
7056 &frame,
7057 "reserved_gate_not_retained",
7058 format!(
7059 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
7060 ),
7061 )?]);
7062 }
7063
7064 let response = ClientControlResponse::SupervisorAck {
7065 module_id,
7066 applied: true,
7067 };
7068 Ok(vec![control_response_body_frame(
7069 &frame,
7070 &response,
7071 "ClientControlResponse::SupervisorAck",
7072 )?])
7073 }
7074
7075 fn plan_rescan(
7079 &self,
7080 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
7081 ) -> Result<RescanPlan, String> {
7082 let mut current = BTreeMap::new();
7083 for module in self.supervisor.list() {
7084 let (spec, health) = module.configuration().map_err(|err| {
7085 format!(
7086 "failed to read configuration for module_id '{}': {err}",
7087 module.module_id()
7088 )
7089 })?;
7090 let enabled = module
7091 .status()
7092 .map_err(|err| {
7093 format!(
7094 "failed to read status for module_id '{}': {err}",
7095 module.module_id()
7096 )
7097 })?
7098 .enabled;
7099 current.insert(
7100 module.module_id().to_string(),
7101 (module, spec, health, enabled),
7102 );
7103 }
7104 let configured = configured_modules
7105 .into_iter()
7106 .map(|module| (module.module_id.clone(), module))
7107 .collect::<BTreeMap<_, _>>();
7108
7109 let added = configured
7110 .keys()
7111 .filter(|module_id| !current.contains_key(*module_id))
7112 .cloned()
7113 .collect::<Vec<_>>();
7114 let removed = current
7115 .keys()
7116 .filter(|module_id| !configured.contains_key(*module_id))
7117 .cloned()
7118 .collect::<Vec<_>>();
7119 let mut changed_pending_reload = Vec::new();
7120 let mut configuration_changes = BTreeSet::new();
7121 let mut enabled_changes = BTreeSet::new();
7122 let mut unchanged = 0_u32;
7123
7124 for (module_id, configured_module) in &configured {
7125 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
7126 else {
7127 continue;
7128 };
7129 let launch_changed = *current_spec != configured_module.module_spec();
7133 let configuration_changed =
7134 launch_changed || *current_health != configured_module.health;
7135 let enabled_changed = *current_enabled != configured_module.enabled
7138 || self
7139 .capability_evaluator
7140 .configured_enabled(module_id)
7141 .is_some_and(|enabled| enabled != configured_module.enabled);
7142 if configuration_changed {
7143 configuration_changes.insert(module_id.clone());
7144 }
7145 if launch_changed {
7146 changed_pending_reload.push(module_id.clone());
7147 }
7148 if enabled_changed {
7149 enabled_changes.insert(module_id.clone());
7150 }
7151 if !configuration_changed && !enabled_changed {
7152 unchanged = unchanged.saturating_add(1);
7153 }
7154 }
7155 Ok(RescanPlan {
7156 current,
7157 configured,
7158 added,
7159 removed,
7160 changed_pending_reload,
7161 configuration_changes,
7162 enabled_changes,
7163 unchanged,
7164 })
7165 }
7166
7167 async fn reconcile_supervised_modules(
7176 &self,
7177 supervisor: &Supervisor,
7178 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
7179 preview: bool,
7180 requester: Principal,
7181 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
7182 let plan = self.plan_rescan(configured_modules)?;
7183
7184 if preview {
7190 return Ok((plan.preview_result(), Vec::new()));
7191 }
7192 let changed = plan
7193 .configuration_changes
7194 .union(&plan.enabled_changes)
7195 .cloned()
7196 .collect::<Vec<_>>();
7197 let RescanPlan {
7198 current,
7199 configured,
7200 added,
7201 removed,
7202 changed_pending_reload,
7203 configuration_changes,
7204 enabled_changes,
7205 unchanged,
7206 } = plan;
7207 if !(added.is_empty()
7212 && removed.is_empty()
7213 && configuration_changes.is_empty()
7214 && enabled_changes.is_empty())
7215 {
7216 if let Some(context) = &self.rescan {
7217 context
7218 .generation
7219 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
7220 }
7221 }
7222
7223 for module_id in &removed {
7224 let module = ¤t
7225 .get(module_id)
7226 .expect("removed module came from current supervisor state")
7227 .0;
7228 module
7229 .retire_requested(Some(requester.clone()))
7230 .await
7231 .map_err(|err| {
7232 format!("failed to retire module_id '{module_id}' during rescan: {err}")
7233 })?;
7234 self.supervisor.record_rescan_removal(module_id);
7268 self.supervisor.retire(module_id);
7269 self.route_outages.forget(module_id);
7270 }
7271
7272 for module_id in configured.keys() {
7273 let Some((module, _, _, current_enabled)) = current.get(module_id) else {
7274 continue;
7275 };
7276 let configured_module = configured
7277 .get(module_id)
7278 .expect("configured module id came from configured map");
7279 if configuration_changes.contains(module_id) {
7280 module
7281 .update_configuration(
7282 configured_module.module_spec(),
7283 configured_module.health.clone(),
7284 configured_module.drain_timeout_ms,
7285 )
7286 .await
7287 .map_err(|err| {
7288 format!(
7289 "failed to update module_id '{module_id}' configuration during rescan: {err}"
7290 )
7291 })?;
7292 }
7293 if *current_enabled != configured_module.enabled {
7296 self.route_outages.mark_operator_action(module_id);
7299 module
7300 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
7301 .await
7302 .map_err(|err| {
7303 self.route_outages.operator_action_ended_unrefused(module_id);
7304 format!(
7305 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
7306 configured_module.enabled
7307 )
7308 })?;
7309 }
7310 }
7311
7312 for module_id in &added {
7313 let configured_module = configured
7314 .get(module_id)
7315 .expect("added module id came from configured map");
7316 supervisor
7317 .supervise_configured_requested(
7318 configured_module.module_spec(),
7319 configured_module.enabled,
7320 configured_module.health.clone(),
7321 configured_module.drain_timeout_ms,
7322 configured_module.restart,
7323 configured_module.register_timeout,
7324 Some(requester.clone()),
7325 )
7326 .map_err(|err| {
7327 format!("failed to add module_id '{module_id}' during rescan: {err}")
7328 })?;
7329 }
7330
7331 Ok((
7332 SupervisorRescanResult {
7333 added,
7334 removed,
7335 changed_pending_reload,
7336 enabled_changes: enabled_changes.iter().cloned().collect(),
7337 unchanged,
7338 preview: false,
7339 restart_required: Vec::new(),
7342 capability_warnings: Vec::new(),
7343 module_actions: Vec::new(),
7344 earlier_pending: Vec::new(),
7345 expect: None,
7346 },
7347 changed,
7348 ))
7349 }
7350
7351 async fn handle_supervisor_set_enabled(
7352 &self,
7353 frame: Frame,
7354 module_id: String,
7355 enabled: bool,
7356 requester: Principal,
7357 ) -> Result<Vec<Frame>, RouterError> {
7358 let operation_lock = self.supervisor.operation_lock();
7359 let _operation_guard = operation_lock.lock().await;
7360 let Some(module) = self.supervisor.get(&module_id) else {
7361 return Ok(vec![control_error_frame(
7362 &frame,
7363 "unknown_module",
7364 format!("module_id '{module_id}' is not supervised"),
7365 )?]);
7366 };
7367
7368 self.route_outages.mark_operator_action(&module_id);
7371 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
7372 Ok(applied) => applied,
7373 Err(err) => {
7374 self.route_outages
7375 .operator_action_ended_unrefused(&module_id);
7376 return Ok(vec![control_error_frame(
7377 &frame,
7378 "target_unavailable",
7379 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
7380 )?]);
7381 }
7382 };
7383 if !applied {
7384 self.route_outages
7387 .operator_action_ended_unrefused(&module_id);
7388 }
7389
7390 self.capability_evaluator.wake_deadline_loop();
7391 self.refresh_capability_requirements();
7392 let response = ClientControlResponse::SupervisorAck { module_id, applied };
7393 Ok(vec![control_response_body_frame(
7394 &frame,
7395 &response,
7396 "ClientControlResponse::SupervisorAck",
7397 )?])
7398 }
7399
7400 async fn handle_supervisor_health_probe(
7401 &self,
7402 ctx: &RouteCtx,
7403 frame: Frame,
7404 module_id: String,
7405 ) -> Result<Vec<Frame>, RouterError> {
7406 self.refresh_capability_requirements();
7407 let Some(registration) = self
7408 .registry
7409 .get_module(&module_id)
7410 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
7411 else {
7412 return Ok(vec![control_error_frame(
7413 &frame,
7414 "unknown_module",
7415 format!("module_id '{module_id}' is not registered"),
7416 )?]);
7417 };
7418
7419 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
7432 {
7433 return Ok(vec![control_error_frame(
7434 &frame,
7435 "health_not_advertised",
7436 format!("module_id '{module_id}' did not advertise health.check"),
7437 )?]);
7438 }
7439
7440 self.observe_control_wait(
7441 ctx,
7442 Some(®istration.manifest.module_id),
7443 ops::SUPERVISOR_HEALTH_PROBE,
7444 async {
7445 let deadline = Instant::now() + self.health_probe_timeout;
7446 let pending = match self.forwarding.begin_module_control_rpc_for(
7447 &module_id,
7448 MODULE_CONTROL_OP_HEALTH_CHECK,
7449 deadline,
7450 ) {
7451 Ok(pending) => pending,
7452 Err(err) => {
7453 return Ok(vec![control_error_frame(
7454 &frame,
7455 forwarding_error_code(&err),
7456 err.to_string(),
7457 )?])
7458 }
7459 };
7460
7461 let PendingModuleControlRpc {
7462 endpoint,
7463 module_sink,
7464 negotiated_ver,
7465 corr: probe_corr,
7466 receiver,
7467 } = pending;
7468 let mut guard =
7469 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
7470 let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
7471 .map_err(|err| {
7472 RouterError::backend(
7473 0,
7474 frame.header.corr,
7475 format!("failed to encode health.check request: {err}"),
7476 )
7477 })?;
7478 let probe_frame = Frame::build_with_version(
7479 negotiated_ver,
7480 FrameType::Request,
7481 control_flags(),
7482 0,
7483 0,
7484 probe_corr,
7485 probe_body,
7486 )
7487 .map_err(RouterError::FrameBuild)?;
7488
7489 match timeout_at(deadline, module_sink.send(probe_frame)).await {
7492 Ok(Ok(())) => {}
7493 Ok(Err(err)) => {
7494 return Ok(vec![control_error_frame(
7495 &frame,
7496 "target_unavailable",
7497 err.to_string(),
7498 )?]);
7499 }
7500 Err(_) => {
7501 return Ok(vec![control_error_frame(
7502 &frame,
7503 "module_timeout",
7504 format!(
7505 "module_id '{module_id}' did not answer health.check within {:?}",
7506 self.health_probe_timeout
7507 ),
7508 )?]);
7509 }
7510 }
7511
7512 match timeout_at(deadline, receiver).await {
7513 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
7514 guard.disarm();
7515 let Some(report) = response.health_report() else {
7516 return Ok(vec![control_error_frame(
7517 &frame,
7518 "invalid_control_body",
7519 "health.check RPC returned a non-health response",
7520 )?]);
7521 };
7522 let HealthReport {
7527 status,
7528 detail,
7529 metrics,
7530 } = report;
7531 let capability_detail = self
7532 .capability_evaluator
7533 .required_problem_detail(&module_id);
7534 let response = ClientControlResponse::SupervisorHealthProbe {
7535 module_id,
7536 status,
7537 detail: append_capability_problem_detail(detail, capability_detail),
7538 metrics,
7539 };
7540 Ok(vec![control_response_body_frame(
7541 &frame,
7542 &response,
7543 "ClientControlResponse::SupervisorHealthProbe",
7544 )?])
7545 }
7546 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
7547 guard.disarm();
7548 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
7553 }
7554 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
7555 guard.disarm();
7556 Ok(vec![control_error_frame(
7557 &frame,
7558 "target_unavailable",
7559 message,
7560 )?])
7561 }
7562 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
7563 guard.disarm();
7564 Ok(vec![control_error_frame(
7565 &frame,
7566 "invalid_control_body",
7567 message,
7568 )?])
7569 }
7570 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
7571 guard.disarm();
7572 Ok(vec![control_error_frame(
7573 &frame,
7574 "invalid_control_body",
7575 format!("expected module-control op '{expected}', got '{actual}'"),
7576 )?])
7577 }
7578 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
7579 guard.disarm();
7580 Ok(vec![control_error_frame(
7581 &frame,
7582 "module_timeout",
7583 format!(
7584 "module_id '{module_id}' answered health.check after {:?}",
7585 self.health_probe_timeout
7586 ),
7587 )?])
7588 }
7589 Ok(Err(_)) => Ok(vec![control_error_frame(
7590 &frame,
7591 "target_unavailable",
7592 "health.check waiter was canceled before the module responded",
7593 )?]),
7594 Err(_) => Ok(vec![control_error_frame(
7595 &frame,
7596 "module_timeout",
7597 format!(
7598 "module_id '{module_id}' did not answer health.check within {:?}",
7599 self.health_probe_timeout
7600 ),
7601 )?]),
7602 }
7603 },
7604 )
7605 .await
7606 }
7607
7608 fn supervisor_status(
7609 &self,
7610 module_id: &str,
7611 corr: u64,
7612 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
7613 self.supervisor
7614 .get(module_id)
7615 .map(|module| {
7616 let warming = module.is_warming_for_control("status").map_err(|err| {
7617 RouterError::backend(
7618 0,
7619 corr,
7620 format!(
7621 "failed to read supervisor warming state for module_id '{module_id}': {err}"
7622 ),
7623 )
7624 })?;
7625 module.status_for_control("status").map_err(|err| {
7626 RouterError::backend(
7627 0,
7628 corr,
7629 format!(
7630 "failed to read supervisor status for module_id '{module_id}': {err}"
7631 ),
7632 )
7633 }).map(|status| (status, warming))
7634 })
7635 .transpose()
7636 }
7637
7638 fn guard_module_control_op(
7639 &self,
7640 frame: &Frame,
7641 module_id: &str,
7642 op: &str,
7643 ) -> Result<Option<Frame>, RouterError> {
7644 if self.module_grants_op(module_id, op, frame.header.corr)? {
7645 return Ok(None);
7646 }
7647
7648 Ok(Some(control_error_frame(
7649 frame,
7650 "op_not_allowed",
7651 format!("module_id '{module_id}' did not grant control op '{op}'"),
7652 )?))
7653 }
7654
7655 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
7656 let Some(registration) = self
7657 .registry
7658 .get_module(module_id)
7659 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
7660 else {
7661 return Ok(false);
7662 };
7663 Ok(module_registration_grants_op(®istration.control_ops, op))
7664 }
7665
7666 fn handle_status_update(
7667 &self,
7668 endpoint: ModuleEndpointId,
7669 frame: Frame,
7670 ) -> Result<Vec<Frame>, RouterError> {
7671 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
7672 Ok(update) => update,
7673 Err(err) => {
7674 if is_known_module_push_op(&frame.body) {
7679 return Ok(vec![control_error_frame(
7680 &frame,
7681 "invalid_control_body",
7682 format!("malformed module control push body: {err}"),
7683 )?]);
7684 }
7685 return Ok(Vec::new());
7686 }
7687 };
7688
7689 match update {
7690 ModuleControlPush::RouteStatus {
7691 route_channel,
7692 route_epoch,
7693 status,
7694 } => {
7695 self.forwarding
7696 .cache_status(endpoint, route_channel, route_epoch, status)
7697 .map_err(RouterError::Forwarding)?;
7698 }
7699 }
7700 Ok(Vec::new())
7701 }
7702
7703 fn handle_route_poll(
7704 &self,
7705 ctx: &RouteCtx,
7706 frame: Frame,
7707 route_channel: u16,
7708 route_epoch: u32,
7709 kind: PollKind,
7710 ) -> Result<Vec<Frame>, RouterError> {
7711 let snapshot = self
7712 .forwarding
7713 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7714 .map_err(RouterError::Forwarding)?;
7715 let response = match (kind, snapshot) {
7716 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7717 ClientControlResponse::RoutePoll {
7718 route_channel,
7719 route_epoch,
7720 status,
7721 live: None,
7722 }
7723 }
7724 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7725 route_channel,
7726 route_epoch,
7727 status: None,
7728 live: None,
7729 },
7730 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7731 let live = self
7757 .process_liveness
7758 .as_ref()
7759 .and_then(|source| source.process_live(&module_id))
7760 .unwrap_or(true);
7761 ClientControlResponse::RoutePoll {
7762 route_channel,
7763 route_epoch,
7764 status: None,
7765 live: Some(live),
7766 }
7767 }
7768 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7769 route_channel,
7770 route_epoch,
7771 status: None,
7772 live: Some(false),
7773 },
7774 };
7775
7776 Ok(vec![control_response_body_frame(
7777 &frame,
7778 &response,
7779 "ClientControlResponse::RoutePoll",
7780 )?])
7781 }
7782
7783 pub(crate) fn observe_module_control_completion(
7784 &self,
7785 completion: ModuleControlRpcCompletion,
7786 ) -> bool {
7787 match completion {
7788 ModuleControlRpcCompletion::Unknown => false,
7789 ModuleControlRpcCompletion::Settled => true,
7790 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7791 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7792 info!(
7793 module_id = %module_id,
7794 latency_ms,
7795 "late health.check answer proves the module is alive"
7796 );
7797 match self
7798 .supervisor
7799 .record_late_health_answer(&module_id, latency_ms)
7800 {
7801 Ok(true) => {}
7802 Ok(false) => debug!(
7803 module_id = %module_id,
7804 latency_ms,
7805 "late health.check answer has no active supervisor snapshot"
7806 ),
7807 Err(err) => warn!(
7808 module_id = %module_id,
7809 latency_ms,
7810 error = %err,
7811 "failed to record late health.check answer"
7812 ),
7813 }
7814 true
7815 }
7816 }
7817 }
7818
7819 fn refuse_to_end_module_connection_for_a_client(
7840 &self,
7841 module_connection_id: ConnectionId,
7842 corr: u64,
7843 err: ForwardingError,
7844 ) -> Result<(), RouterError> {
7845 if let ForwardingError::ConnectionClosing { connection_id } = err {
7846 if connection_id != module_connection_id {
7847 warn!(
7848 module_connection_id = module_connection_id.get(),
7849 client_connection_id = connection_id.get(),
7850 corr,
7851 "dropping a route.bind response for a closing client; the module connection keeps serving"
7852 );
7853 return Ok(());
7854 }
7855 }
7856 Err(RouterError::Forwarding(err))
7857 }
7858
7859 fn handle_module_relay_response(
7860 &self,
7861 connection_id: ConnectionId,
7862 frame: Frame,
7863 ) -> Result<Vec<Frame>, RouterError> {
7864 let mut secondary_error = None;
7865 let outcome = match frame.header.ty {
7866 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7867 Ok(probe) if probe.op == "route.bind" => {
7868 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7869 Ok(ModuleControlResponse::RouteBindAck {}) => {
7870 RouteBindRelayOutcome::Accepted
7871 }
7872 Ok(other) => {
7873 let message =
7874 format!("route.bind response carried unexpected body: {other:?}");
7875 secondary_error = Some(control_error_frame(
7876 &frame,
7877 "invalid_control_body",
7878 message.clone(),
7879 )?);
7880 RouteBindRelayOutcome::ModuleGone(message)
7881 }
7882 Err(err) => {
7883 let message = format!("malformed route.bind response body: {err}");
7884 secondary_error = Some(control_error_frame(
7885 &frame,
7886 "invalid_control_body",
7887 message.clone(),
7888 )?);
7889 RouteBindRelayOutcome::ModuleGone(message)
7890 }
7891 }
7892 }
7893 Ok(probe) => {
7894 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7895 {
7896 Ok(response) => ModuleControlRpcOutcome::Response(response),
7897 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7898 "malformed {} response body: {err}",
7899 probe.op
7900 )),
7901 };
7902 let completion = self
7903 .forwarding
7904 .complete_module_control_rpc(
7905 connection_id,
7906 frame.header.corr,
7907 Some(&probe.op),
7908 outcome,
7909 )
7910 .map_err(RouterError::Forwarding)?;
7911 if !self.observe_module_control_completion(completion) {
7912 debug!(
7913 connection_id = connection_id.get(),
7914 corr = frame.header.corr,
7915 op = %probe.op,
7916 "dropping late or unknown module-control RPC response"
7917 );
7918 }
7919 return Ok(Vec::new());
7920 }
7921 Err(err) => {
7922 if let Some(expected_op) = self
7923 .forwarding
7924 .pending_module_control_op(connection_id, frame.header.corr)
7925 .map_err(RouterError::Forwarding)?
7926 {
7927 let completion = self
7928 .forwarding
7929 .complete_module_control_rpc(
7930 connection_id,
7931 frame.header.corr,
7932 None,
7933 ModuleControlRpcOutcome::MalformedResponse(format!(
7934 "malformed {expected_op} response body: {err}"
7935 )),
7936 )
7937 .map_err(RouterError::Forwarding)?;
7938 if !self.observe_module_control_completion(completion) {
7939 debug!(
7940 connection_id = connection_id.get(),
7941 corr = frame.header.corr,
7942 "dropping late malformed module-control RPC response"
7943 );
7944 }
7945 return Ok(Vec::new());
7946 }
7947 let message = format!("malformed route.bind response body: {err}");
7948 secondary_error = Some(control_error_frame(
7949 &frame,
7950 "invalid_control_body",
7951 message.clone(),
7952 )?);
7953 RouteBindRelayOutcome::ModuleGone(message)
7954 }
7955 },
7956 FrameType::Error => {
7957 if self
7958 .forwarding
7959 .pending_module_control_op(connection_id, frame.header.corr)
7960 .map_err(RouterError::Forwarding)?
7961 .is_some()
7962 {
7963 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7964 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7965 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7966 "malformed module-control ERROR body: {err}"
7967 )),
7968 };
7969 let completion = self
7970 .forwarding
7971 .complete_module_control_rpc(
7972 connection_id,
7973 frame.header.corr,
7974 None,
7975 outcome,
7976 )
7977 .map_err(RouterError::Forwarding)?;
7978 if !self.observe_module_control_completion(completion) {
7979 debug!(
7980 connection_id = connection_id.get(),
7981 corr = frame.header.corr,
7982 "dropping late or unknown module-control RPC error"
7983 );
7984 }
7985 return Ok(Vec::new());
7986 }
7987 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7988 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7989 Err(err) => {
7990 let message = format!("malformed route.bind ERROR body: {err}");
7991 secondary_error = Some(control_error_frame(
7992 &frame,
7993 "invalid_control_body",
7994 message.clone(),
7995 )?);
7996 RouteBindRelayOutcome::ModuleGone(message)
7997 }
7998 }
7999 }
8000 ty => {
8001 return Ok(vec![control_error_frame(
8002 &frame,
8003 "unsupported_control_frame",
8004 format!("unsupported module channel-0 frame {ty:?}"),
8005 )?])
8006 }
8007 };
8008
8009 let settled =
8010 self.forwarding
8011 .complete_pending_relay(connection_id, frame.header.corr, outcome);
8012 let completion = match settled {
8013 Ok(completion) => completion,
8014 Err(err) => {
8015 self.refuse_to_end_module_connection_for_a_client(
8016 connection_id,
8017 frame.header.corr,
8018 err,
8019 )?;
8020 return Ok(secondary_error.into_iter().collect());
8021 }
8022 };
8023 if let Some(target) = completion.abandoned.as_ref() {
8024 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
8025 }
8026 if !completion.settled {
8027 debug!(
8028 connection_id = connection_id.get(),
8029 corr = frame.header.corr,
8030 frame_type = ?frame.header.ty,
8031 "dropping late or unknown route.bind relay response"
8032 );
8033 }
8034 Ok(secondary_error.into_iter().collect())
8035 }
8036
8037 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
8038 debug!(connection_id = connection_id.get(), "handling GOODBYE");
8039 self.cleanup_connection_with_end_reason(
8043 connection_id,
8044 RegistrationEndReason::ExplicitGoodbye,
8045 )
8046 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
8047 Ok(Vec::new())
8048 }
8049}
8050
8051impl Default for ControlHandler {
8052 fn default() -> Self {
8053 Self::new(Arc::new(Registry::default()))
8054 }
8055}
8056
8057impl crate::supervise::SwapPromotionObserver for ControlHandler {
8058 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
8059 self.apply_registration_capabilities(registration);
8060 }
8061}
8062
8063fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
8064 CapabilityRequirementStatus {
8065 consumer: status.consumer,
8066 capability: status.capability,
8067 need: match status.need {
8068 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
8069 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
8070 },
8071 verdict: status.verdict.as_str().to_string(),
8072 episode_seq: status.episode_seq,
8073 config_satisfiable: status.config_satisfiable,
8074 runtime_available: status.runtime_available,
8075 detail: status.detail,
8076 }
8077}
8078
8079fn append_capability_problem_detail(
8080 detail: Option<String>,
8081 capability_detail: Option<String>,
8082) -> Option<String> {
8083 match (detail, capability_detail) {
8084 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
8085 (Some(detail), None) => Some(detail),
8086 (None, Some(capability_detail)) => Some(capability_detail),
8087 (None, None) => None,
8088 }
8089}
8090
8091fn subc_ops() -> Vec<String> {
8092 SUBC_CONTROL_OPS
8093 .iter()
8094 .map(|op| (*op).to_string())
8095 .collect()
8096}
8097
8098fn module_subc_ops() -> Vec<String> {
8099 SUBC_CONTROL_OPS
8100 .iter()
8101 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
8102 .map(|op| (*op).to_string())
8103 .collect()
8104}
8105
8106#[cfg(test)]
8107fn module_baseline_control_ops() -> Vec<String> {
8108 MODULE_BASELINE_CONTROL_OPS
8109 .iter()
8110 .map(|op| (*op).to_string())
8111 .collect()
8112}
8113
8114fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
8115 let mut seen = HashSet::new();
8116 let mut effective = Vec::new();
8117 for op in MODULE_BASELINE_CONTROL_OPS {
8118 if seen.insert((*op).to_string()) {
8119 effective.push((*op).to_string());
8120 }
8121 }
8122 for op in declared.unwrap_or_default() {
8123 if seen.insert(op.clone()) {
8124 effective.push(op);
8125 }
8126 }
8127 effective
8128}
8129
8130fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
8131 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
8132}
8133
8134fn target_module_id(target: &RouteTarget) -> &str {
8135 match target {
8136 RouteTarget::ToolProvider { module_id }
8137 | RouteTarget::ManagementSurface { module_id }
8138 | RouteTarget::InternalService { module_id, .. } => module_id,
8139 }
8140}
8141
8142fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
8143 roles.iter().any(|role| match (target, role) {
8144 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
8145 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
8146 (
8147 RouteTarget::InternalService { service_id, .. },
8148 ProviderRole::InternalService {
8149 service_id: provided,
8150 ..
8151 },
8152 ) => service_id == provided,
8153 _ => false,
8154 })
8155}
8156
8157fn is_routable_role(role: &ProviderRole) -> bool {
8158 matches!(
8159 role,
8160 ProviderRole::ToolProvider { .. }
8161 | ProviderRole::ManagementSurface { .. }
8162 | ProviderRole::InternalService { .. }
8163 )
8164}
8165
8166#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8167enum ControlRequestBodyError {
8168 UnknownOp,
8169 InvalidBody,
8170}
8171
8172#[derive(Debug, Deserialize)]
8173struct ControlOpProbe {
8174 op: String,
8175}
8176
8177const MODULE_PUSH_OPS: &[&str] = &["route.status"];
8180
8181fn is_known_module_push_op(body: &[u8]) -> bool {
8182 serde_json::from_slice::<ControlOpProbe>(body)
8183 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
8184 .unwrap_or(false)
8185}
8186
8187fn is_known_module_request_op(body: &[u8]) -> bool {
8188 serde_json::from_slice::<ControlOpProbe>(body)
8189 .map(|probe| is_module_to_subc_op(&probe.op))
8190 .unwrap_or(false)
8191}
8192
8193fn is_module_to_subc_op(op: &str) -> bool {
8194 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
8195}
8196
8197fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
8198 debug!(
8199 op = %op,
8200 connection_id = connection_id.get(),
8201 corr,
8202 "control dispatch"
8203 );
8204}
8205
8206fn log_slow_control_dispatch(
8207 dispatch_started_at: Option<StdInstant>,
8208 op: &'static str,
8209 connection_id: ConnectionId,
8210 corr: u64,
8211 route_open_timings: Option<&RouteOpenTimings>,
8212) {
8213 let Some(dispatch_started_at) = dispatch_started_at else {
8214 return;
8215 };
8216 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
8218 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
8219 if let Some(timings) = route_open_timings {
8220 warn!(
8221 op = %op,
8222 connection_id = connection_id.get(),
8223 corr,
8224 elapsed_ms = elapsed.as_millis() as u64,
8225 module_id = ?timings.module_id,
8226 principal = %timings.principal,
8227 admission_ms = timings.admission.as_millis() as u64,
8228 project_root_ms = timings.project_root.as_millis() as u64,
8229 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
8230 bind_send_ms = timings.bind_send.as_millis() as u64,
8231 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
8232 "slow control dispatch"
8233 );
8234 return;
8235 }
8236 warn!(
8237 op = %op,
8238 connection_id = connection_id.get(),
8239 corr,
8240 elapsed_ms = elapsed.as_millis() as u64,
8241 "slow control dispatch"
8242 );
8243 }
8244}
8245
8246#[cfg(test)]
8248type TestCallerLookup =
8249 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
8250
8251fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
8253 match request {
8254 ClientControlRequest::SupervisorRestart { module_id, .. } => {
8255 Some(PrivilegedOperation::Restart { module_id })
8256 }
8257 ClientControlRequest::SupervisorSwap { module_id, .. } => {
8258 Some(PrivilegedOperation::Swap { module_id })
8259 }
8260 ClientControlRequest::SupervisorReload { module_id } => {
8261 Some(PrivilegedOperation::Reload { module_id })
8262 }
8263 ClientControlRequest::SupervisorRescan { preview } => {
8264 if *preview {
8265 None
8266 } else {
8267 Some(PrivilegedOperation::Rescan)
8268 }
8269 }
8270 ClientControlRequest::SupervisorRescanChecked {
8275 preview,
8276 candidate,
8277 expect,
8278 } => (!*preview && candidate.is_none() && expect.is_some())
8279 .then_some(PrivilegedOperation::Rescan),
8280 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
8281 Some(PrivilegedOperation::SetEnabled {
8282 module_id,
8283 enabled: *enabled,
8284 })
8285 }
8286 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
8287 Some(PrivilegedOperation::ReleaseReserved { module_id })
8288 }
8289 ClientControlRequest::ServerDescribe {}
8290 | ClientControlRequest::DaemonShutdownIntent { .. }
8293 | ClientControlRequest::CatalogList { .. }
8294 | ClientControlRequest::RouteOpen { .. }
8295 | ClientControlRequest::RoutePoll { .. }
8296 | ClientControlRequest::SupervisorList {}
8297 | ClientControlRequest::SupervisorHealth {}
8298 | ClientControlRequest::SupervisorHealthProbe { .. }
8299 | ClientControlRequest::SupervisorRoutes { .. }
8300 | ClientControlRequest::SupervisorProvenance { .. }
8301 | ClientControlRequest::SupervisorStderrTail { .. }
8302 | ClientControlRequest::SupervisorTerminals { .. }
8303 | ClientControlRequest::SupervisorSpawnSnapshot {}
8304 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
8305 }
8306}
8307
8308fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
8309 match request {
8310 ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
8311 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
8312 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
8313 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
8314 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
8315 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
8316 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
8317 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
8318 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
8319 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
8320 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
8321 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
8322 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
8323 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
8324 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
8325 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
8326 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
8327 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
8328 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
8329 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
8330 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
8331 }
8332}
8333
8334struct RouteOpenCaller {
8342 principal: String,
8346 scope: String,
8349 harness: String,
8352}
8353
8354impl RouteOpenCaller {
8355 fn new(
8356 principal: Option<&Principal>,
8357 scope: Option<&ScopeSelector>,
8358 identity: &BindIdentity,
8359 ) -> Self {
8360 Self {
8361 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
8362 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
8363 harness: bounded_log_text(&identity.harness),
8364 }
8365 }
8366
8367 fn unknown() -> Self {
8370 Self {
8371 principal: "unknown".to_string(),
8372 scope: "unknown".to_string(),
8373 harness: "unknown".to_string(),
8374 }
8375 }
8376}
8377
8378const MAX_LOGGED_TEXT_CHARS: usize = 64;
8381
8382fn bounded_log_text(value: &str) -> String {
8386 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
8387 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
8388 bounded.push('…');
8389 }
8390 format!("{bounded:?}")
8391}
8392
8393fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
8399 use subc_protocol::scope::ScopeKind;
8400 let kind = match stamp.kind {
8401 ScopeKind::Head => "head",
8402 ScopeKind::Worker => "worker",
8403 ScopeKind::Ephemeral => "ephemeral",
8404 ScopeKind::Federated => "federated",
8405 _ => "unknown",
8406 };
8407 format!(
8408 "{kind}/{}/{}@{}",
8409 principal_label(&stamp.owner),
8410 bounded_log_text(&stamp.scope_ref),
8411 stamp.scope_epoch
8412 )
8413}
8414
8415fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
8422 let epoch = selector
8423 .scope_epoch
8424 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
8425 format!(
8426 "requested/{}/{}@{epoch}",
8427 bounded_log_text(&principal_label(&selector.owner)),
8428 bounded_log_text(&selector.scope_ref),
8429 )
8430}
8431
8432fn principal_label(principal: &Principal) -> String {
8433 match principal {
8434 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
8435 Principal::Direct => "direct".to_string(),
8436 other => format!("{other:?}"),
8437 }
8438}
8439
8440fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
8441 match request {
8442 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
8443 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
8444 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
8445 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
8446 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
8447 ModuleControlRequestFromModule::RouteAuthorize { .. } => {
8448 subc_protocol::scope::ROUTE_AUTHORIZE_OP
8449 }
8450 ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
8451 _ => "unsupported",
8452 }
8453}
8454
8455fn parse_client_control_request(
8456 body: &[u8],
8457) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
8458 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
8459 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8460 Ok(probe)
8461 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
8462 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
8463 {
8464 ControlRequestBodyError::InvalidBody
8465 }
8466 Ok(_) => ControlRequestBodyError::UnknownOp,
8467 Err(_) => ControlRequestBodyError::InvalidBody,
8468 };
8469 (err, classification)
8470 })
8471}
8472
8473fn parse_module_control_request_from_module(
8474 body: &[u8],
8475) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
8476 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
8477 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8478 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
8479 Ok(_) => ControlRequestBodyError::UnknownOp,
8480 Err(_) => ControlRequestBodyError::InvalidBody,
8481 };
8482 (err, classification)
8483 })
8484}
8485
8486#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
8487enum ProviderRoleKind {
8488 ToolProvider,
8489 PipelineStage,
8490 ManagementSurface,
8491 InternalService,
8492}
8493
8494fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
8495 match role {
8496 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
8497 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
8498 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
8499 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
8500 }
8501}
8502
8503fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
8504 roles.iter().map(provider_role_kind).collect()
8505}
8506
8507const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
8510
8511#[derive(Debug, Default, PartialEq, Eq)]
8513struct ScopeOutcomeCounts {
8514 created: usize,
8515 replaced: usize,
8516 updated: usize,
8517 unchanged: usize,
8518 refused: usize,
8519}
8520
8521impl ScopeOutcomeCounts {
8522 fn of(results: &[ScopeRecordResult]) -> Self {
8523 let mut counts = Self::default();
8524 for result in results {
8525 let slot = match result.outcome {
8526 ScopeRecordOutcome::Created => &mut counts.created,
8527 ScopeRecordOutcome::Replaced => &mut counts.replaced,
8528 ScopeRecordOutcome::Updated => &mut counts.updated,
8529 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
8530 ScopeRecordOutcome::Refused => &mut counts.refused,
8531 };
8532 *slot += 1;
8533 }
8534 counts
8535 }
8536}
8537
8538#[cfg(test)]
8539mod scope_outcome_count_tests {
8540 use super::*;
8541
8542 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
8543 ScopeRecordResult {
8544 scope_ref: "r".to_string(),
8545 scope_epoch: 1,
8546 outcome,
8547 code: None,
8548 message: None,
8549 version: None,
8550 parent_state: None,
8551 }
8552 }
8553
8554 #[test]
8557 fn every_outcome_is_counted_in_its_own_field() {
8558 let results = [
8559 result(ScopeRecordOutcome::Created),
8560 result(ScopeRecordOutcome::Created),
8561 result(ScopeRecordOutcome::Replaced),
8562 result(ScopeRecordOutcome::Updated),
8563 result(ScopeRecordOutcome::Unchanged),
8564 result(ScopeRecordOutcome::Refused),
8565 result(ScopeRecordOutcome::Refused),
8566 result(ScopeRecordOutcome::Refused),
8567 ];
8568 assert_eq!(
8569 ScopeOutcomeCounts::of(&results),
8570 ScopeOutcomeCounts {
8571 created: 2,
8572 replaced: 1,
8573 updated: 1,
8574 unchanged: 1,
8575 refused: 3,
8576 }
8577 );
8578 }
8579}
8580
8581fn capability_census_trigger(
8585 old: Option<&CapabilityDeclarations>,
8586 new: Option<&CapabilityDeclarations>,
8587) -> bool {
8588 let old_provides = old
8589 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
8590 .unwrap_or_default();
8591 let old_denies = old
8592 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
8593 .unwrap_or_default();
8594 let new = new.cloned().unwrap_or(CapabilityDeclarations {
8595 provides: Vec::new(),
8596 requires: Vec::new(),
8597 must_never_reach: Vec::new(),
8598 });
8599
8600 new.provides
8601 .iter()
8602 .any(|capability| !old_provides.contains(capability))
8603 || new
8604 .must_never_reach
8605 .iter()
8606 .any(|capability| !old_denies.contains(capability))
8607}
8608
8609fn catalog_update_frozen_field_message(
8610 registered: &ModuleManifest,
8611 provides: &[ProviderRole],
8612) -> Option<String> {
8613 let old_has_provides = !registered.provides.is_empty();
8614 let new_has_provides = !provides.is_empty();
8615 if old_has_provides != new_has_provides {
8616 return Some(format!(
8617 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
8618 registered.module_id
8619 ));
8620 }
8621
8622 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
8623 return Some(format!(
8624 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
8625 registered.module_id
8626 ));
8627 }
8628
8629 let registered_concurrency = manifest_concurrency(registered);
8630 let mut candidate = registered.clone();
8631 candidate.provides = provides.to_vec();
8632 let candidate_concurrency = manifest_concurrency(&candidate);
8633 if candidate_concurrency != registered_concurrency {
8634 return Some(format!(
8635 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
8636 registered.module_id, registered_concurrency, candidate_concurrency
8637 ));
8638 }
8639
8640 None
8643}
8644
8645fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
8646 manifest.provides.iter().any(is_routable_role)
8647}
8648
8649fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
8655 manifest
8656 .provides
8657 .iter()
8658 .find_map(|provider| match provider {
8659 ProviderRole::ToolProvider { concurrency, .. }
8660 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
8661 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
8662 })
8663 .unwrap_or(Concurrency::ModuleManaged)
8664}
8665
8666fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
8673 let has_management_surface = manifest
8674 .provides
8675 .iter()
8676 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
8677 if !has_management_surface {
8678 return false;
8679 }
8680 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
8681 return false;
8682 };
8683 let Some(provides) = raw
8684 .get("manifest")
8685 .and_then(|manifest| manifest.get("provides"))
8686 .and_then(serde_json::Value::as_array)
8687 else {
8688 return false;
8689 };
8690 provides.iter().any(|role| {
8695 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8696 && role.get("concurrency").is_none()
8697 })
8698}
8699
8700fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8701 if peer_version != PROTOCOL_VERSION {
8702 return Err(format!(
8703 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8704 ));
8705 }
8706 Ok(PROTOCOL_VERSION)
8707}
8708
8709fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8710 Frame::build_with_version(
8711 response_version(frame),
8712 FrameType::Pong,
8713 frame.header.flags,
8714 0,
8715 0,
8716 frame.header.corr,
8717 Vec::new(),
8718 )
8719 .map_err(RouterError::FrameBuild)
8720}
8721
8722pub(crate) fn control_error_frame(
8723 frame: &Frame,
8724 code: &'static str,
8725 message: impl Into<String>,
8726) -> Result<Frame, RouterError> {
8727 control_error_body_frame(
8728 frame,
8729 ErrorBody {
8730 code: code.to_string(),
8731 message: message.into(),
8732 detail: None,
8733 },
8734 )
8735}
8736
8737fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8745 control_error_frame_with_flags(frame, error, daemon_error_flags())
8746}
8747
8748fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8754 control_error_frame_with_flags(frame, error, control_flags())
8755}
8756
8757fn control_error_frame_with_flags(
8758 frame: &Frame,
8759 error: ErrorBody,
8760 flags: Flags,
8761) -> Result<Frame, RouterError> {
8762 let body = serde_json::to_vec(&error).map_err(|err| {
8763 RouterError::backend(
8764 0,
8765 frame.header.corr,
8766 format!("failed to encode control ERROR: {err}"),
8767 )
8768 })?;
8769
8770 Frame::build_with_version(
8771 response_version(frame),
8772 FrameType::Error,
8773 flags,
8774 0,
8775 0,
8776 frame.header.corr,
8777 body,
8778 )
8779 .map_err(RouterError::FrameBuild)
8780}
8781
8782pub(crate) fn control_response_body_frame<T: Serialize>(
8783 frame: &Frame,
8784 reply: &T,
8785 label: &'static str,
8786) -> Result<Frame, RouterError> {
8787 let body = serde_json::to_vec(reply).map_err(|err| {
8788 RouterError::backend(
8789 0,
8790 frame.header.corr,
8791 format!("failed to encode {label}: {err}"),
8792 )
8793 })?;
8794
8795 Frame::build_with_version(
8796 response_version(frame),
8797 FrameType::Response,
8798 control_flags(),
8799 0,
8800 0,
8801 frame.header.corr,
8802 body,
8803 )
8804 .map_err(RouterError::FrameBuild)
8805}
8806
8807fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8837 match err {
8838 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8839 ForwardingError::NoModuleConnection => "target_unavailable",
8840 ForwardingError::ModuleReloading { .. } => "module_reloading",
8841 ForwardingError::ClientRouteChannelExhausted { .. }
8842 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8843 ForwardingError::StaleModuleEndpoint
8844 | ForwardingError::UnknownReservation { .. }
8845 | ForwardingError::ConnectionClosing { .. }
8846 | ForwardingError::ClientEgressClosed { .. }
8847 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8848 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8851 ForwardingError::RelayCorrelationExhausted
8852 | ForwardingError::RouteOpenBuild(_)
8853 | ForwardingError::Poisoned => "forwarding_error",
8854 }
8855}
8856
8857fn response_version(frame: &Frame) -> u8 {
8858 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8859 frame.header.ver
8860 } else {
8861 PROTOCOL_VERSION
8862 }
8863}
8864
8865fn control_flags() -> Flags {
8866 Flags::new(false, Priority::Passive, false)
8867}
8868
8869fn send_goodbye_target_best_effort(
8874 counters: &DaemonCounters,
8875 target: &GoodbyeTarget,
8876 context: &'static str,
8877) {
8878 let Ok(frame) = Frame::build_with_version(
8879 target.negotiated_ver,
8880 FrameType::Goodbye,
8881 control_flags(),
8882 target.channel,
8883 target.epoch,
8884 0,
8885 Vec::new(),
8886 ) else {
8887 return;
8888 };
8889 crate::forwarding::send_module_route_goodbye(
8890 counters,
8891 &target.sink,
8892 frame,
8893 target.module_id.as_deref(),
8894 context,
8895 );
8896}
8897
8898pub(crate) fn send_route_control_pushes(
8899 forwarding: &ForwardingTable,
8900 routes: Vec<EndpointRoute>,
8901 push: ClientControlPush,
8902) {
8903 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8904 for route in routes {
8905 let target = route.goodbye_target;
8906 if let Some((existing, channels)) = targets
8907 .iter_mut()
8908 .find(|(existing, _)| existing.connection_id == target.connection_id)
8909 {
8910 debug_assert_eq!(
8911 existing.negotiated_ver, target.negotiated_ver,
8912 "one connection cannot negotiate multiple frame versions"
8913 );
8914 if !channels.contains(&target.channel) {
8915 channels.push(target.channel);
8916 }
8917 continue;
8918 }
8919 let channel = target.channel;
8920 targets.push((target, vec![channel]));
8921 }
8922 for (target, mut channels) in targets {
8923 channels.sort_unstable();
8924 let mut push = push.clone();
8925 match &mut push {
8926 ClientControlPush::RouteClosing {
8927 channels: covered, ..
8928 }
8929 | ClientControlPush::RouteClosed {
8930 channels: covered, ..
8931 } => *covered = channels,
8932 }
8933 let body = match serde_json::to_vec(&push) {
8934 Ok(body) => body,
8935 Err(err) => {
8936 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8937 continue;
8938 }
8939 };
8940 let frame = match Frame::build_with_version(
8941 target.negotiated_ver,
8942 FrameType::Push,
8943 control_flags(),
8944 0,
8945 0,
8946 0,
8947 body.clone(),
8948 ) {
8949 Ok(frame) => frame,
8950 Err(err) => {
8951 warn!(
8952 route_channel = target.channel,
8953 error = %err,
8954 "failed to build route lifecycle control PUSH frame"
8955 );
8956 continue;
8957 }
8958 };
8959 if let Err(err) = target.sink.try_send(frame) {
8960 if target.close_on_delivery_failure() {
8961 warn!(
8962 target_connection_id = target.connection_id.get(),
8963 route_channel = target.channel,
8964 error = %err,
8965 "route lifecycle control PUSH was not delivered to client; closing target connection"
8966 );
8967 let _ = forwarding.escalate_client_delivery_failure(
8968 target.connection_id,
8969 target.channel,
8970 target.epoch,
8971 CloseReason::new(
8972 "route_lifecycle_push_delivery_failed",
8973 format!(
8974 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8975 target.channel
8976 ),
8977 ),
8978 crate::forwarding::UndeliveredFrame {
8979 module_id: target.module_id.as_deref(),
8980 sink: &target.sink,
8981 },
8982 );
8983 }
8984 }
8985 }
8986}
8987
8988#[cfg(test)]
8989mod tests {
8990 include!("control/agent_commands_tests.rs");
8991 include!("control/resource_revision_tests.rs");
8992 #[tokio::test]
8993 async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
8994 let registry = Arc::new(Registry::default());
8995 let handler = ControlHandler::new(registry);
8996 assert_eq!(
8997 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8998 None
8999 );
9000 let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
9001 hello_via_sink(
9002 &handler,
9003 &ctx,
9004 &mut rx,
9005 hello_frame("daemon", PROTOCOL_VERSION, 1),
9006 )
9007 .await;
9008 assert_eq!(
9009 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
9010 Some("daemon".into())
9011 );
9012 assert_eq!(
9013 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
9014 None
9015 );
9016 assert_eq!(
9017 handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
9018 None,
9019 "a registered module is not necessarily supervised"
9020 );
9021 assert_eq!(
9022 handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
9023 None,
9024 "aggregate provenance has no single target"
9025 );
9026 assert_eq!(
9027 handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
9028 None
9029 );
9030 }
9031
9032 mod peer_manifest {
9033 include!("peer_manifest_tests.rs");
9034 }
9035 mod activity_read {
9036 include!("activity_read_tests.rs");
9037 }
9038
9039 mod lifecycle_activity {
9040 include!("control_lifecycle_tests.rs");
9041 }
9042 #[cfg(unix)]
9043 #[tokio::test]
9044 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
9045 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
9046 let path = dir.join("subc.jsonc");
9047 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
9048 "program":"/bin/sleep","args":["60"],"protocol":"none",
9049 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
9050 }}}).to_string()).unwrap();
9051 let mut configured = crate::daemon_config::load(&path)
9052 .unwrap()
9053 .unwrap()
9054 .modules
9055 .pop()
9056 .unwrap();
9057 let registry = std::sync::Arc::new(crate::Registry::default());
9058 let handle = crate::SupervisorHandle::new();
9059 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
9060 .with_handle(handle.clone());
9061 let module = supervisor
9062 .supervise_configured_with_health(
9063 configured.module_spec(),
9064 true,
9065 configured.health.clone(),
9066 None,
9067 configured.restart,
9068 )
9069 .unwrap();
9070 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
9071 let before = module.status().unwrap().pid;
9072 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
9073 configured.health.cadence = std::time::Duration::from_secs(3600);
9074 let health_only = handler
9075 .reconcile_supervised_modules(
9076 &supervisor,
9077 vec![configured.clone()],
9078 false,
9079 Principal::Direct,
9080 )
9081 .await
9082 .unwrap();
9083 assert!(
9084 health_only.0.changed_pending_reload.is_empty(),
9085 "health policy is already applied live"
9086 );
9087 assert_eq!(module.status().unwrap().pid, before);
9088 assert_eq!(
9089 module.configuration().unwrap().1.http,
9090 configured.health.http
9091 );
9092 configured.args = vec!["61".into()];
9093 let launch = handler
9094 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
9095 .await
9096 .unwrap();
9097 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
9098 assert_eq!(
9099 module.status().unwrap().pid,
9100 before,
9101 "a launch edit is stored until reload"
9102 );
9103 module.drain().await.unwrap();
9104 }
9105 use cortexkit_test_support::ScratchDir;
9106 use std::{
9107 collections::BTreeMap,
9108 fmt,
9109 path::PathBuf,
9110 sync::{Arc, Mutex},
9111 time::Duration,
9112 };
9113
9114 use serde_json::{json, Value};
9115 use subc_protocol::{
9116 manifest::{
9117 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
9118 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
9119 },
9120 session::HealthStatus,
9121 FrameType,
9122 };
9123
9124 use super::*;
9125 use crate::{
9126 forwarding::{DataRoute, DataRouteState},
9127 registry::ChannelState,
9128 router::FrameSink,
9129 stderr_tail::DEFAULT_MAX_LINE_BYTES,
9130 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
9131 RouteCtx, Router,
9132 };
9133 use tokio::{
9134 sync::mpsc,
9135 time::{sleep, Instant},
9136 };
9137 use tracing::{
9138 field::{Field, Visit},
9139 Event, Subscriber,
9140 };
9141 use tracing_subscriber::{layer::Context, prelude::*, Layer};
9142
9143 fn fake_aft_stub_path() -> PathBuf {
9162 let mut path = std::env::current_exe().expect("current_exe available in tests");
9163 path.pop(); path.pop(); path.push(if cfg!(windows) {
9166 "fake-aft-stub.exe"
9167 } else {
9168 "fake-aft-stub"
9169 });
9170 assert!(
9171 path.exists(),
9172 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
9173 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
9174 path.display()
9175 );
9176 path
9177 }
9178
9179 fn client_retries(code: &str) -> bool {
9184 subc_protocol::error_codes::is_retryable_route_open(code)
9185 }
9186
9187 #[test]
9198 fn retryability_of_forwarding_codes_matches_the_failure() {
9199 let transient = [
9202 ForwardingError::NoModuleConnection,
9203 ForwardingError::ModuleReloading {
9204 module_id: "m".into(),
9205 },
9206 ForwardingError::StaleModuleEndpoint,
9207 ForwardingError::UnknownReservation {
9208 client_channel: 1,
9209 module_channel: 1,
9210 },
9211 ForwardingError::ConnectionClosing {
9212 connection_id: ConnectionId::new(1),
9213 },
9214 ForwardingError::ClientEgressClosed {
9215 connection_id: ConnectionId::new(1),
9216 },
9217 ForwardingError::ModuleEgressUnavailable {
9218 connection_id: ConnectionId::new(1),
9219 },
9220 ];
9221 for err in transient {
9222 let code = forwarding_error_code(&err);
9223 assert!(
9224 client_retries(code),
9225 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
9226 );
9227 }
9228
9229 let permanent = [
9234 ForwardingError::ConnectionRoleConflict {
9235 connection_id: ConnectionId::new(1),
9236 },
9237 ForwardingError::ClientRouteChannelExhausted {
9238 connection_id: ConnectionId::new(1),
9239 },
9240 ForwardingError::ModuleRouteChannelExhausted {
9241 endpoint: ModuleEndpointId {
9242 connection_id: ConnectionId::new(1),
9243 generation: 1,
9244 },
9245 },
9246 ForwardingError::RelayCorrelationExhausted,
9247 ForwardingError::RouteOpenBuild("x".into()),
9248 ForwardingError::Poisoned,
9249 ];
9250 for err in permanent {
9251 let code = forwarding_error_code(&err);
9252 assert!(
9253 !client_retries(code),
9254 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
9255 );
9256 }
9257 }
9258
9259 #[tokio::test]
9287 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
9288 let handler = ControlHandler::default();
9289 let frame =
9290 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
9291
9292 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
9296 assert_eq!(
9297 stamped,
9298 Principal::Direct,
9299 "a caller that proved nothing must not be stamped as a supervised module"
9300 );
9301
9302 let forged = handler
9307 .route_open_principal(
9308 &frame,
9309 Some(ConsumerIdentity {
9310 module_id: "aft".to_string(),
9311 launch_nonce: "not-a-real-nonce".to_string(),
9312 }),
9313 )
9314 .unwrap();
9315 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
9316 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
9317 }
9318
9319 #[test]
9336 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
9337 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
9338 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
9339 let ClientControlRequest::RouteOpen {
9340 consumer_identity, ..
9341 } = parsed
9342 else {
9343 panic!("route.open body must parse as RouteOpen");
9344 };
9345 assert_eq!(
9346 consumer_identity,
9347 Some(ConsumerIdentity {
9348 module_id: "aft".to_string(),
9349 launch_nonce: "n".to_string(),
9350 }),
9351 "the wire field name must reach the value route_open_principal reads"
9352 );
9353 }
9354
9355 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
9356 ModuleManifest::builder(module_id, "0.1.0")
9357 .protocol_ver(protocol_ver)
9358 .provides(vec![ProviderRole::ToolProvider {
9359 tools: vec![Tool {
9360 name: "read".to_string(),
9361 description: None,
9362 execution_mode: ExecutionMode::Pure,
9363 schema: json!({"type": "object"}),
9364 }],
9365 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
9366 concurrency: Concurrency::ModuleManaged,
9367 emits_push: true,
9368 sub_supervises: true,
9369 }])
9370 .build()
9371 }
9372
9373 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
9374 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
9375 }
9376
9377 fn hello_frame_with_control_ops(
9378 module_id: &str,
9379 protocol_ver: u8,
9380 corr: u64,
9381 control_ops: Option<Vec<String>>,
9382 ) -> Frame {
9383 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
9384 }
9385
9386 fn hello_frame_with_nonce(
9387 module_id: &str,
9388 protocol_ver: u8,
9389 corr: u64,
9390 launch_nonce: Option<&str>,
9391 ) -> Frame {
9392 hello_frame_full(
9393 module_id,
9394 protocol_ver,
9395 corr,
9396 None,
9397 launch_nonce.map(ToOwned::to_owned),
9398 )
9399 }
9400
9401 fn hello_frame_full(
9402 module_id: &str,
9403 protocol_ver: u8,
9404 corr: u64,
9405 control_ops: Option<Vec<String>>,
9406 launch_nonce: Option<String>,
9407 ) -> Frame {
9408 let body = serde_json::to_vec(&ModuleHelloBody {
9409 manifest: manifest(module_id, protocol_ver),
9410 protocol_ver,
9411 control_ops,
9412 launch_nonce,
9413 })
9414 .unwrap();
9415 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9416 }
9417
9418 fn hello_frame_with_rich_ui(
9419 module_id: &str,
9420 protocol_ver: u8,
9421 corr: u64,
9422 launch_nonce: Option<&str>,
9423 rich_ui: Value,
9424 ) -> Frame {
9425 let mut body = serde_json::to_value(ModuleHelloBody {
9426 manifest: manifest(module_id, protocol_ver),
9427 protocol_ver,
9428 control_ops: None,
9429 launch_nonce: launch_nonce.map(str::to_string),
9430 })
9431 .unwrap();
9432 body["manifest"]["rich_ui"] = rich_ui;
9433 Frame::build(
9434 FrameType::Hello,
9435 control_flags(),
9436 0,
9437 0,
9438 corr,
9439 serde_json::to_vec(&body).unwrap(),
9440 )
9441 .unwrap()
9442 }
9443
9444 fn non_routable_hello_frame_with_control_ops(
9445 module_id: &str,
9446 corr: u64,
9447 control_ops: Option<Vec<String>>,
9448 ) -> Frame {
9449 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9450 manifest.provides.clear();
9451 let body = serde_json::to_vec(&ModuleHelloBody {
9452 manifest,
9453 protocol_ver: PROTOCOL_VERSION,
9454 control_ops,
9455 launch_nonce: None,
9456 })
9457 .unwrap();
9458 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9459 }
9460
9461 fn capability_grammar_hello_frame(
9462 capabilities: Value,
9463 runtime_computed: Option<Value>,
9464 corr: u64,
9465 ) -> Frame {
9466 let mut body = serde_json::to_value(ModuleHelloBody {
9467 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
9468 protocol_ver: PROTOCOL_VERSION,
9469 control_ops: None,
9470 launch_nonce: None,
9471 })
9472 .expect("HELLO body serializes");
9473 body["manifest"]["capabilities"] = capabilities;
9474 if let Some(runtime_computed) = runtime_computed {
9475 body["runtime_computed"] = runtime_computed;
9476 }
9477 Frame::build(
9478 FrameType::Hello,
9479 control_flags(),
9480 0,
9481 0,
9482 corr,
9483 serde_json::to_vec(&body).expect("HELLO body reserializes"),
9484 )
9485 .expect("HELLO frame builds")
9486 }
9487
9488 fn channel_request(channel: u16, corr: u64) -> Frame {
9489 Frame::build(
9490 FrameType::Request,
9491 Flags::new(true, Priority::Interactive, false),
9492 channel,
9493 0,
9494 corr,
9495 b"opaque".to_vec(),
9496 )
9497 .unwrap()
9498 }
9499
9500 fn route_ctx(
9501 connection_id: ConnectionId,
9502 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
9503 let (tx, rx) = mpsc::channel(8);
9504 (
9505 RouteCtx {
9506 connection_id,
9507 egress: FrameSink::new(tx),
9508 },
9509 rx,
9510 )
9511 }
9512
9513 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
9514 serde_json::from_slice(&frame.body).unwrap()
9515 }
9516
9517 async fn hello_via_sink(
9522 handler: &ControlHandler,
9523 ctx: &RouteCtx,
9524 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9525 hello: Frame,
9526 ) -> Frame {
9527 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
9528 assert!(
9529 replies.is_empty(),
9530 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
9531 );
9532 let ack = rx
9533 .try_recv()
9534 .expect("HELLO_ACK is queued on the module sink")
9535 .frame;
9536 assert_eq!(ack.header.ty, FrameType::HelloAck);
9537 ack
9538 }
9539
9540 fn parse_error(frame: &Frame) -> Value {
9541 serde_json::from_slice(&frame.body).unwrap()
9542 }
9543
9544 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
9545 serde_json::from_slice(&frame.body).unwrap()
9546 }
9547
9548 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
9549 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
9550 route_channel,
9551 route_epoch: 0,
9552 kind,
9553 })
9554 .unwrap();
9555 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9556 }
9557
9558 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
9559 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
9560 module_id: module_id.to_string(),
9561 })
9562 .unwrap();
9563 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9564 }
9565
9566 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
9567 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
9568 }
9569
9570 fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
9572 let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
9573 let mut frame = route_open_frame(corr, module_id, root);
9574 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
9575 body["identity"]["project_root"] = serde_json::json!(path.as_path());
9576 frame.body = serde_json::to_vec(&body).unwrap();
9577 frame
9578 }
9579
9580 fn route_open_frame_with_consumer_capabilities(
9581 corr: u64,
9582 module_id: &str,
9583 _project_root: ScratchDir,
9584 consumer_capabilities: Option<Vec<String>>,
9585 ) -> Frame {
9586 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9587 busy_refusals: None,
9588 target: RouteTarget::ToolProvider {
9589 module_id: module_id.to_string(),
9590 },
9591 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
9592 consumer_identity: None,
9593 consumer_capabilities,
9594 role_versions: None,
9595 admission_facts: None,
9596 scope: None,
9597 })
9598 .unwrap();
9599 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9600 }
9601
9602 fn route_open_frame_with_role_versions(
9603 corr: u64,
9604 module_id: &str,
9605 _project_root: ScratchDir,
9606 role_versions: Option<BTreeMap<String, String>>,
9607 ) -> Frame {
9608 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9609 busy_refusals: None,
9610 target: RouteTarget::ToolProvider {
9611 module_id: module_id.to_string(),
9612 },
9613 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9614 consumer_identity: None,
9615 consumer_capabilities: None,
9616 role_versions,
9617 admission_facts: None,
9618 scope: None,
9619 })
9620 .unwrap();
9621 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9622 }
9623
9624 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
9625 entries
9626 .iter()
9627 .map(|(role, version)| (role.to_string(), version.to_string()))
9628 .collect()
9629 }
9630
9631 fn route_open_frame_with_admission_facts(
9632 corr: u64,
9633 module_id: &str,
9634 _project_root: ScratchDir,
9635 consumer_identity: Option<subc_control::ConsumerIdentity>,
9636 facts: Option<Value>,
9637 ) -> Frame {
9638 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9639 busy_refusals: None,
9640 target: RouteTarget::ToolProvider {
9641 module_id: module_id.to_string(),
9642 },
9643 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9644 consumer_identity,
9645 consumer_capabilities: None,
9646 role_versions: None,
9647 admission_facts: facts,
9648 scope: None,
9649 })
9650 .unwrap();
9651 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9652 }
9653
9654 #[derive(Clone, Default)]
9655 struct EventCapture {
9656 events: Arc<Mutex<Vec<CapturedEvent>>>,
9657 }
9658
9659 #[derive(Clone, Debug)]
9660 struct CapturedEvent {
9661 target: String,
9662 level: tracing::Level,
9663 fields: BTreeMap<String, String>,
9664 }
9665
9666 impl EventCapture {
9667 fn events(&self) -> Vec<CapturedEvent> {
9668 self.events.lock().unwrap().clone()
9669 }
9670 }
9671
9672 impl<S> Layer<S> for EventCapture
9673 where
9674 S: Subscriber,
9675 {
9676 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
9677 let mut visitor = EventFieldVisitor::default();
9678 event.record(&mut visitor);
9679 self.events.lock().unwrap().push(CapturedEvent {
9680 target: event.metadata().target().to_string(),
9681 level: *event.metadata().level(),
9682 fields: visitor.fields,
9683 });
9684 }
9685 }
9686
9687 #[derive(Default)]
9688 struct EventFieldVisitor {
9689 fields: BTreeMap<String, String>,
9690 }
9691
9692 impl Visit for EventFieldVisitor {
9693 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
9694 self.fields
9695 .insert(field.name().to_string(), format!("{value:?}"));
9696 }
9697 }
9698
9699 fn health_response(corr: u64, status: HealthStatus) -> Frame {
9700 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
9701 status,
9702 detail: Some("warming".to_string()),
9703 metrics: Some(json!({"queue_depth": 3})),
9704 })
9705 .unwrap();
9706 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9707 }
9708
9709 fn route_bind_ack(corr: u64) -> Frame {
9710 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
9711 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9712 }
9713
9714 fn unique_project_root(label: &str) -> ScratchDir {
9715 ScratchDir::new(label)
9716 }
9717
9718 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
9719 match parse_route_poll(frame) {
9720 ClientControlResponse::RoutePoll {
9721 status: None,
9722 live: Some(live),
9723 ..
9724 } => assert_eq!(live, expected_live),
9725 other => panic!("unexpected route.poll response: {other:?}"),
9726 }
9727 }
9728
9729 fn bind_liveness_route(
9730 registry: &Registry,
9731 forwarding: &ForwardingTable,
9732 module_id: &str,
9733 ) -> (RouteCtx, u16, u32) {
9734 let module_connection = ConnectionId::new(101);
9735 let client_connection = ConnectionId::new(202);
9736 let registration = registry
9737 .register_with_control_ops(
9738 manifest(module_id, PROTOCOL_VERSION),
9739 PROTOCOL_VERSION,
9740 module_connection,
9741 module_baseline_control_ops(),
9742 )
9743 .unwrap();
9744 let (module_tx, _module_rx) = mpsc::channel(8);
9745 let endpoint = forwarding
9746 .register_module_connection(
9747 module_connection,
9748 module_id.to_string(),
9749 PROTOCOL_VERSION,
9750 manifest_concurrency(®istration.manifest),
9751 FrameSink::new(module_tx),
9752 )
9753 .unwrap();
9754 let (client_ctx, _client_rx) = route_ctx(client_connection);
9755 let pending = forwarding
9756 .begin_route_bind_relay_for_test(
9757 client_connection,
9758 client_ctx.egress.clone(),
9759 1,
9760 module_id,
9761 )
9762 .unwrap();
9763 assert_eq!(pending.endpoint, endpoint);
9764 let route_channel = pending.client_channel;
9765 let route_epoch = pending.client_epoch;
9766 forwarding
9767 .complete_pending_relay(
9768 module_connection,
9769 pending.corr,
9770 RouteBindRelayOutcome::Accepted,
9771 )
9772 .unwrap();
9773 (client_ctx, route_channel, route_epoch)
9774 }
9775
9776 struct FakeProcessLiveness {
9777 live: Option<bool>,
9778 }
9779
9780 impl ModuleProcessLiveness for FakeProcessLiveness {
9781 fn process_live(&self, _module_id: &str) -> Option<bool> {
9782 self.live
9783 }
9784 }
9785
9786 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9787 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9788 {
9789 let registry = Arc::new(Registry::default());
9790 let supervisor_handle = SupervisorHandle::new();
9791 let supervisor = Supervisor::new_for_test(
9792 Arc::clone(®istry),
9793 RestartPolicy::new(1, Duration::from_millis(10)),
9794 )
9795 .with_handle(supervisor_handle.clone());
9796 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9797 let module = supervisor
9798 .spawn(ModuleSpec {
9799 carry_containment: false,
9800 module_id: "stderr-tail-wire".to_string(),
9801 program: fake_aft_stub_path(),
9802 args: Vec::new(),
9803 env: vec![
9804 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9805 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9806 ],
9807 reserved: false,
9808 reserved_prefixes: Vec::new(),
9809 protocol: ModuleProtocol::Subc,
9810 overlap: Default::default(),
9811 })
9812 .unwrap();
9813
9814 let deadline = Instant::now() + Duration::from_secs(5);
9815 loop {
9816 let tail = module.stderr_tail(None, None);
9817 if tail
9818 .entries
9819 .iter()
9820 .any(|entry| matches!(entry, TailEntry::ProcessStart))
9821 && tail.entries.iter().any(|entry| {
9822 matches!(
9823 entry,
9824 TailEntry::Line {
9825 truncated: true,
9826 ..
9827 }
9828 )
9829 })
9830 {
9831 break;
9832 }
9833 assert!(
9834 Instant::now() < deadline,
9835 "module did not produce a truncated line and restart boundary: {tail:?}"
9836 );
9837 sleep(Duration::from_millis(10)).await;
9838 }
9839
9840 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9841 let request = ClientControlRequest::SupervisorStderrTail {
9842 module_id: "stderr-tail-wire".to_string(),
9843 max_lines: None,
9844 max_bytes: None,
9845 };
9846 let frame = Frame::build(
9847 FrameType::Request,
9848 control_flags(),
9849 0,
9850 0,
9851 1,
9852 serde_json::to_vec(&request).unwrap(),
9853 )
9854 .unwrap();
9855 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9856 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9857 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9858 serde_json::from_slice(&responses[0].body).unwrap()
9859 else {
9860 panic!("expected supervisor.stderr_tail response");
9861 };
9862
9863 assert!(
9864 tail.entries
9865 .iter()
9866 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9867 "the control response lost the restart boundary"
9868 );
9869 let Some(StderrTailEntry::Line {
9870 text,
9871 truncated,
9872 at_ms,
9873 }) = tail.entries.iter().find(|entry| {
9874 matches!(
9875 entry,
9876 StderrTailEntry::Line {
9877 truncated: true,
9878 ..
9879 }
9880 )
9881 })
9882 else {
9883 panic!("the control response lost the truncated line");
9884 };
9885 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9886 assert!(*truncated);
9887 assert!(
9888 at_ms.is_some(),
9889 "the control response lost the line's capture time"
9890 );
9891 }
9892
9893 #[tokio::test(flavor = "current_thread")]
9898 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9899 let dir = ScratchDir::new("terminals-off-worker");
9900 let journal_path = dir.join("terminals.jsonl");
9901 let registry = Arc::new(Registry::default());
9902 let supervisor_handle = SupervisorHandle::new();
9903 let supervisor =
9904 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9905 .with_handle(supervisor_handle.clone())
9906 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9907 let module = supervisor
9908 .spawn(ModuleSpec {
9909 carry_containment: false,
9910 module_id: "terminal-off-worker".to_string(),
9911 program: fake_aft_stub_path(),
9912 args: Vec::new(),
9913 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9914 reserved: false,
9915 reserved_prefixes: Vec::new(),
9916 protocol: ModuleProtocol::Subc,
9917 overlap: Default::default(),
9918 })
9919 .unwrap();
9920 let deadline = Instant::now() + Duration::from_secs(5);
9921 while module.terminal_history().entries.len() != 2 {
9922 assert!(Instant::now() < deadline, "module did not record two exits");
9923 sleep(Duration::from_millis(10)).await;
9924 }
9925
9926 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9927 let handler =
9928 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
9929 let frame = Frame::build(
9930 FrameType::Request,
9931 control_flags(),
9932 0,
9933 0,
9934 1,
9935 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9936 module_id: "terminal-off-worker".to_string(),
9937 })
9938 .unwrap(),
9939 )
9940 .unwrap();
9941 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9942 let spawned_at = std::time::Instant::now();
9943 let read = tokio::spawn({
9944 let handler = Arc::clone(&handler);
9945 async move { handler.handle_control_frame(&ctx, frame).await }
9946 });
9947 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9950 .await
9951 .unwrap()
9952 .expect("the history read reached its pause");
9953 let elapsed = spawned_at.elapsed();
9954 assert!(
9955 elapsed < Duration::from_secs(2) && !read.is_finished(),
9956 "this task could not run while the history read was paused \
9957 (resumed after {elapsed:?}, read finished: {})",
9958 read.is_finished()
9959 );
9960
9961 drop(release);
9962 let responses = read.await.unwrap().unwrap();
9963 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9964 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9965 panic!("expected supervisor.terminals response");
9966 };
9967 assert_eq!(terminals.entries.len(), 2);
9968 assert_eq!(terminals.journal_skipped_lines, 0);
9969 assert_eq!(terminals.journal_read_errors, 0);
9970 }
9971
9972 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9973 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9974 let registry = Arc::new(Registry::default());
9975 let supervisor_handle = SupervisorHandle::new();
9976 let supervisor =
9977 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9978 .with_handle(supervisor_handle.clone());
9979 let module = supervisor
9980 .spawn(ModuleSpec {
9981 carry_containment: false,
9982 module_id: "terminal-golden".to_string(),
9983 program: fake_aft_stub_path(),
9984 args: Vec::new(),
9985 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9986 reserved: false,
9987 reserved_prefixes: Vec::new(),
9988 protocol: ModuleProtocol::Subc,
9989 overlap: Default::default(),
9990 })
9991 .unwrap();
9992
9993 let deadline = Instant::now() + Duration::from_secs(5);
9994 while module.terminal_history().entries.len() != 2 {
9995 assert!(
9996 Instant::now() < deadline,
9997 "module did not retain two terminal exits: {:?}",
9998 module.terminal_history()
9999 );
10000 sleep(Duration::from_millis(10)).await;
10001 }
10002
10003 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10004 let request = ClientControlRequest::SupervisorTerminals {
10005 module_id: "terminal-golden".to_string(),
10006 };
10007 let frame = Frame::build(
10008 FrameType::Request,
10009 control_flags(),
10010 0,
10011 0,
10012 1,
10013 serde_json::to_vec(&request).unwrap(),
10014 )
10015 .unwrap();
10016 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
10017 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
10018 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
10019 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
10020 panic!("expected supervisor.terminals response");
10021 };
10022 assert_eq!(terminals.entries.len(), 2);
10023 assert_eq!(terminals.dropped, 0);
10024
10025 let mut rendered = serde_json::to_value(response).unwrap();
10026 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
10029 for (index, entry) in rendered["entries"]
10030 .as_array_mut()
10031 .expect("terminal response entries array")
10032 .iter_mut()
10033 .enumerate()
10034 {
10035 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
10036 }
10037
10038 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10039 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
10040 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
10041 if std::env::var_os("UPDATE_GOLDEN").is_some() {
10042 std::fs::write(&golden_path, &serialized).unwrap();
10043 }
10044 let expected: Value =
10045 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
10046 assert_eq!(rendered, expected);
10047 }
10048
10049 #[test]
10050 fn hello_registers_manifest_and_returns_ack() {
10051 let registry = Arc::new(Registry::default());
10052 let handler = ControlHandler::new(Arc::clone(®istry));
10053 let conn = ConnectionId::new(1);
10054
10055 let responses = handler
10056 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
10057 .unwrap();
10058
10059 assert_eq!(responses.len(), 1);
10060 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10061 assert_eq!(responses[0].header.channel, 0);
10062 assert_eq!(responses[0].header.corr, 7);
10063 let ack = parse_ack(&responses[0]);
10064 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
10065 assert!(ack
10066 .subc_capabilities
10067 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
10068 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
10069 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
10070 assert!(ack
10071 .subc_ops
10072 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
10073 assert!(ack
10074 .subc_ops
10075 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
10076
10077 let registration = registry.get_module("aft").unwrap().unwrap();
10078 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
10079 assert_eq!(registration.state, ChannelState::Active);
10080 assert_eq!(registration.connection_id, conn);
10081 assert_eq!(registration.control_ops, module_baseline_control_ops());
10082 assert!(registration.manifest.display_name.is_none());
10083 }
10084
10085 #[test]
10086 fn hello_display_name_refuses_invalid_manifest_by_field() {
10087 let registry = Arc::new(Registry::default());
10088 let handler = ControlHandler::new(Arc::clone(®istry));
10089 for (index, invalid) in [
10090 json!(""),
10091 json!("a".repeat(41)),
10092 json!(" Routing"),
10093 json!("a\nb"),
10094 json!("a\u{202e}b"),
10095 json!(7),
10096 ]
10097 .into_iter()
10098 .enumerate()
10099 {
10100 let module_id = format!("bad-display-name-{index}");
10101 let mut body = serde_json::to_value(ModuleHelloBody {
10102 manifest: manifest(&module_id, PROTOCOL_VERSION),
10103 protocol_ver: PROTOCOL_VERSION,
10104 control_ops: None,
10105 launch_nonce: None,
10106 })
10107 .unwrap();
10108 body["manifest"]["display_name"] = invalid;
10109 let hello = Frame::build(
10110 FrameType::Hello,
10111 control_flags(),
10112 0,
10113 0,
10114 7,
10115 serde_json::to_vec(&body).unwrap(),
10116 )
10117 .unwrap();
10118 let response = handler
10119 .handle_control(ConnectionId::new(100 + index as u64), hello)
10120 .unwrap();
10121 assert_eq!(response[0].header.ty, FrameType::Error);
10122 let error = parse_error(&response[0]);
10123 assert_eq!(error["code"], "invalid_manifest");
10124 assert!(error["message"].as_str().unwrap().contains("display_name"));
10125 assert!(registry.get_module(&module_id).unwrap().is_none());
10126 }
10127 }
10128
10129 #[test]
10130 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
10131 let invalid_identifiers = [
10132 ("case_change", "credentials-Provider/v1"),
10133 ("leading_zero", "credentials-provider/v01"),
10134 ("trailing_hyphen", "credentials-provider-/v1"),
10135 ("consecutive_hyphens", "credentials--provider/v1"),
10136 ("uppercase", "Credentials-provider/v1"),
10137 ("missing_v", "credentials-provider/1"),
10138 ("whitespace", "credentials provider/v1"),
10139 ("zero_version", "credentials-provider/v0"),
10140 ("out_of_range_version", "credentials-provider/v4294967296"),
10141 (
10142 "overlength_name",
10143 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
10144 ),
10145 ];
10146 let mut cases = invalid_identifiers
10147 .into_iter()
10148 .map(|(name, identifier)| {
10149 (
10150 format!("identifier_{name}"),
10151 "capabilities.provides[0]".to_string(),
10152 identifier.to_string(),
10153 json!({ "provides": [identifier] }),
10154 None,
10155 )
10156 })
10157 .collect::<Vec<_>>();
10158 cases.extend([
10159 (
10160 "unknown_need".to_string(),
10161 "capabilities.requires[0].need".to_string(),
10162 "deferred".to_string(),
10163 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
10164 None,
10165 ),
10166 (
10167 "duplicate_provides".to_string(),
10168 "capabilities.provides[1]".to_string(),
10169 "credentials-provider/v1".to_string(),
10170 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
10171 None,
10172 ),
10173 (
10174 "duplicate_must_never_reach".to_string(),
10175 "capabilities.must_never_reach[1]".to_string(),
10176 "credentials-provider/v1".to_string(),
10177 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
10178 None,
10179 ),
10180 (
10181 "duplicate_requires_same_need".to_string(),
10182 "capabilities.requires[1]".to_string(),
10183 "credentials-provider/v1".to_string(),
10184 json!({ "requires": [
10185 { "capability": "credentials-provider/v1", "need": "required" },
10186 { "capability": "credentials-provider/v1", "need": "required" }
10187 ] }),
10188 None,
10189 ),
10190 (
10191 "duplicate_requires_conflicting_need".to_string(),
10192 "capabilities.requires[1]".to_string(),
10193 "credentials-provider/v1".to_string(),
10194 json!({ "requires": [
10195 { "capability": "credentials-provider/v1", "need": "required" },
10196 { "capability": "credentials-provider/v1", "need": "optional" }
10197 ] }),
10198 None,
10199 ),
10200 (
10201 "capabilities_root_pointer".to_string(),
10202 "runtime_computed[0]".to_string(),
10203 "/capabilities".to_string(),
10204 json!({}),
10205 Some(json!(["/capabilities"])),
10206 ),
10207 (
10208 "capabilities_descendant_pointer".to_string(),
10209 "runtime_computed[0]".to_string(),
10210 "/capabilities/provides".to_string(),
10211 json!({}),
10212 Some(json!(["/capabilities/provides"])),
10213 ),
10214 (
10215 "malformed_pointer_without_leading_slash".to_string(),
10216 "runtime_computed[0]".to_string(),
10217 "capabilities".to_string(),
10218 json!({}),
10219 Some(json!(["capabilities"])),
10220 ),
10221 (
10222 "malformed_pointer_escape".to_string(),
10223 "runtime_computed[0]".to_string(),
10224 "/roles/~2/tools".to_string(),
10225 json!({}),
10226 Some(json!(["/roles/~2/tools"])),
10227 ),
10228 (
10229 "unknown_capabilities_field".to_string(),
10230 "capabilities.future".to_string(),
10231 "<array>".to_string(),
10232 json!({ "future": [] }),
10233 None,
10234 ),
10235 ]);
10236
10237 for (index, (name, field, value, capabilities, runtime_computed)) in
10238 cases.into_iter().enumerate()
10239 {
10240 let registry = Arc::new(Registry::default());
10241 let handler = ControlHandler::new(Arc::clone(®istry));
10242 let response = handler
10243 .handle_control(
10244 ConnectionId::new((index + 1) as u64),
10245 capability_grammar_hello_frame(
10246 capabilities,
10247 runtime_computed,
10248 index as u64 + 1,
10249 ),
10250 )
10251 .expect("invalid HELLO returns a refusal");
10252
10253 assert_eq!(response.len(), 1, "{name} must emit one refusal");
10254 let error = parse_error(&response[0]);
10255 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
10256 let message = error["message"]
10257 .as_str()
10258 .expect("error message is a string");
10259 assert!(
10260 message.contains(&field),
10261 "{name}: field missing from {message}"
10262 );
10263 assert!(
10264 message.contains(&value),
10265 "{name}: value missing from {message}"
10266 );
10267 assert_eq!(
10268 registry
10269 .active_registration_count()
10270 .expect("registry reads"),
10271 0,
10272 "{name}: refused HELLO must not create a catalog entry"
10273 );
10274 }
10275 }
10276
10277 #[test]
10278 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
10279 let registry = Arc::new(Registry::default());
10280 let handler = ControlHandler::new(Arc::clone(®istry));
10281 let capabilities = json!({
10282 "provides": ["credentials-provider/v1"],
10283 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
10284 "must_never_reach": ["federation-transport/v1"]
10285 });
10286 let response = handler
10287 .handle_control(
10288 ConnectionId::new(99),
10289 capability_grammar_hello_frame(
10290 capabilities.clone(),
10291 Some(json!(["/roles/0/tools"])),
10292 99,
10293 ),
10294 )
10295 .expect("valid HELLO registers");
10296 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10297
10298 let request = Frame::build(
10299 FrameType::Request,
10300 control_flags(),
10301 0,
10302 0,
10303 100,
10304 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10305 .expect("catalog request serializes"),
10306 )
10307 .expect("catalog request frame builds");
10308 let response = handler
10309 .handle_catalog_list(request, None)
10310 .expect("catalog list succeeds");
10311 let ClientControlResponse::CatalogList { modules, .. } =
10312 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
10313 else {
10314 panic!("catalog request must return catalog.list");
10315 };
10316 assert_eq!(modules.len(), 1);
10317 assert_eq!(
10318 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
10319 capabilities
10320 );
10321 }
10322
10323 #[test]
10324 fn catalog_list_mirrors_management_operation_description() {
10325 let registry = Arc::new(Registry::default());
10326 let handler = ControlHandler::new(Arc::clone(®istry));
10327 let description = "List managed records and return their identifiers and metadata.";
10328 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
10329 manifest.provides = vec![ProviderRole::ManagementSurface {
10330 operations: vec![ManagementOperation {
10331 name: "records.list".to_string(),
10332 kind: ManagementOperationKind::Query,
10333 description: Some(description.to_string()),
10334 }],
10335 config_schema: json!({"type": "object"}),
10336 observability: vec![ObservabilitySurface {
10337 name: "records.stats".to_string(),
10338 kind: ObservabilityKind::Snapshot,
10339 }],
10340 identity_scope: vec![IdentityScope::Project],
10341 concurrency: Concurrency::ModuleManaged,
10342 }];
10343 registry
10344 .register_with_control_ops(
10345 manifest,
10346 PROTOCOL_VERSION,
10347 ConnectionId::new(99),
10348 Vec::new(),
10349 )
10350 .expect("described management manifest registers");
10351
10352 let request = Frame::build(
10353 FrameType::Request,
10354 control_flags(),
10355 0,
10356 0,
10357 100,
10358 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10359 .expect("catalog request serializes"),
10360 )
10361 .expect("catalog request frame builds");
10362 let response = handler
10363 .handle_catalog_list(request, None)
10364 .expect("catalog list succeeds");
10365 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10366 assert_eq!(
10367 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
10368 "catalog.list must preserve the declared operation description verbatim"
10369 );
10370 }
10371
10372 #[test]
10373 fn catalog_list_mirrors_registered_operation_delivery() {
10374 let registry = Arc::new(Registry::default());
10375 let handler = ControlHandler::new(Arc::clone(®istry));
10376 let delivery =
10377 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
10378 let mut body = serde_json::to_value(ModuleHelloBody {
10379 manifest: manifest("watch-management", PROTOCOL_VERSION),
10380 protocol_ver: PROTOCOL_VERSION,
10381 control_ops: None,
10382 launch_nonce: None,
10383 })
10384 .unwrap();
10385 body["manifest"]["provides"] = json!([{
10386 "role": "management_surface",
10387 "operations": [
10388 {"name": "status", "kind": "query"},
10389 {"name": "future", "kind": "query"},
10390 {"name": "reset", "kind": "mutate"}
10391 ],
10392 "config_schema": {}, "observability": [], "identity_scope": []
10393 }]);
10394 body["manifest"]["operation_delivery"] = delivery.clone();
10395 let hello = Frame::build(
10396 FrameType::Hello,
10397 control_flags(),
10398 0,
10399 0,
10400 99,
10401 serde_json::to_vec(&body).unwrap(),
10402 )
10403 .unwrap();
10404 let response = handler
10405 .handle_control(ConnectionId::new(99), hello)
10406 .unwrap();
10407 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10408
10409 let (_, registrations) = registry.list_modules().unwrap();
10410 let registered = ®istrations[0].manifest;
10411 assert!(registered.is_watch_operation("status"));
10412 assert!(!registered.is_watch_operation("future"));
10413 assert!(!registered.is_watch_operation("reset"));
10414 assert!(!registered.is_watch_operation("missing"));
10415
10416 let request = Frame::build(
10417 FrameType::Request,
10418 control_flags(),
10419 0,
10420 0,
10421 100,
10422 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10423 )
10424 .unwrap();
10425 let response = handler.handle_catalog_list(request, None).unwrap();
10426 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
10427 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
10428 let ClientControlResponse::CatalogList { modules, .. } =
10429 serde_json::from_slice(&response[0].body).unwrap()
10430 else {
10431 panic!("expected catalog.list");
10432 };
10433 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
10434 }
10435
10436 #[test]
10437 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
10438 let registry = Arc::new(Registry::default());
10439 let supervisor = SupervisorHandle::new();
10440 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10441 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10442 let descriptor = json!({
10443 "schema": "ckios.richui.manifest/1",
10444 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10445 "bytes": 4096,
10446 "chunks": 4
10447 });
10448 let connection = ConnectionId::new(201);
10449 let hello = handler
10450 .handle_control(
10451 connection,
10452 hello_frame_with_rich_ui(
10453 "vault",
10454 PROTOCOL_VERSION,
10455 201,
10456 Some("the-real-nonce"),
10457 descriptor.clone(),
10458 ),
10459 )
10460 .unwrap();
10461 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10462
10463 let request =
10464 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
10465 let provides = registry
10466 .get_module_by_connection(connection)
10467 .unwrap()
10468 .unwrap()
10469 .manifest
10470 .provides;
10471 handler
10472 .handle_catalog_update(connection, request, provides, None, Some(false))
10473 .unwrap();
10474
10475 let request = Frame::build(
10476 FrameType::Request,
10477 control_flags(),
10478 0,
10479 0,
10480 203,
10481 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10482 )
10483 .unwrap();
10484 let response = handler.handle_catalog_list(request, None).unwrap();
10485 let ClientControlResponse::CatalogList { modules, .. } =
10486 serde_json::from_slice(&response[0].body).unwrap()
10487 else {
10488 panic!("expected catalog.list");
10489 };
10490 assert_eq!(
10491 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
10492 descriptor
10493 );
10494 assert!(modules[0].nonce_verified);
10495 }
10496
10497 fn hello_with_role_declarations(declarations: Value) -> Frame {
10498 let mut body = serde_json::to_value(ModuleHelloBody {
10499 manifest: manifest("role-test", PROTOCOL_VERSION),
10500 protocol_ver: PROTOCOL_VERSION,
10501 control_ops: None,
10502 launch_nonce: None,
10503 })
10504 .unwrap();
10505 body["manifest"]["role_declarations"] = declarations;
10506 Frame::build(
10507 FrameType::Hello,
10508 control_flags(),
10509 0,
10510 0,
10511 301,
10512 serde_json::to_vec(&body).unwrap(),
10513 )
10514 .unwrap()
10515 }
10516
10517 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
10518 let request = Frame::build(
10519 FrameType::Request,
10520 control_flags(),
10521 0,
10522 0,
10523 302,
10524 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10525 )
10526 .unwrap();
10527 let response = handler.handle_catalog_list(request, None).unwrap();
10528 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
10529 let ClientControlResponse::CatalogList { mut modules, .. } =
10530 serde_json::from_slice(&response[0].body).unwrap()
10531 else {
10532 panic!("expected catalog.list");
10533 };
10534 (raw["modules"][0].clone(), modules.remove(0))
10535 }
10536
10537 #[test]
10538 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
10539 let registry = Arc::new(Registry::default());
10540 let handler = ControlHandler::new(Arc::clone(®istry));
10541 let declarations = json!({
10542 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
10543 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
10544 });
10545 let connection = ConnectionId::new(301);
10546 let response = handler
10547 .handle_control(
10548 connection,
10549 hello_with_role_declarations(declarations.clone()),
10550 )
10551 .unwrap();
10552 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10553 let registered = registry
10554 .get_module_by_connection(connection)
10555 .unwrap()
10556 .unwrap()
10557 .manifest;
10558 assert_eq!(
10562 serde_json::to_value(®istered.role_declarations).unwrap(),
10563 declarations
10564 );
10565 let (raw, entry) = role_declarations_catalog(&handler);
10566 assert_eq!(raw["role_declarations"], declarations);
10567 assert_eq!(entry.role_declarations, registered.role_declarations);
10568
10569 let request =
10570 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
10571 handler
10572 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
10573 .unwrap();
10574 let (raw, entry) = role_declarations_catalog(&handler);
10575 assert_eq!(raw["role_declarations"], declarations);
10576 assert_eq!(entry.role_declarations, registered.role_declarations);
10577 }
10578
10579 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
10580 let registry = Arc::new(Registry::default());
10581 let handler = ControlHandler::new(Arc::clone(®istry));
10582 let capture = EventCapture::default();
10583 let _subscriber =
10584 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10585 let response = handler
10586 .handle_control(
10587 ConnectionId::new(301),
10588 hello_with_role_declarations(declarations),
10589 )
10590 .unwrap();
10591 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10592 assert!(registry
10593 .get_module_by_connection(ConnectionId::new(301))
10594 .unwrap()
10595 .unwrap()
10596 .manifest
10597 .role_declarations
10598 .is_empty());
10599 let (raw, entry) = role_declarations_catalog(&handler);
10600 assert!(raw.get("role_declarations").is_none());
10601 assert!(entry.role_declarations.is_empty());
10602 let warnings: Vec<_> = capture
10603 .events()
10604 .into_iter()
10605 .filter(|event| {
10606 event.fields.get("message").map(String::as_str)
10607 == Some("dropping whole role_declarations map from HELLO")
10608 })
10609 .collect();
10610 assert_eq!(warnings.len(), 1);
10611 assert_eq!(warnings[0].level, tracing::Level::WARN);
10612 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
10613 assert_eq!(warnings[0].fields["reason"], reason);
10614 }
10615
10616 #[test]
10617 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
10618 for value in [
10619 json!(null),
10620 json!(false),
10621 json!(0),
10622 json!("opaque"),
10623 json!([]),
10624 ] {
10625 assert_hello_drops_role_declarations_with_warning(
10626 json!({"routing/v1": {}, "broken/v1": value}),
10627 "role_declarations values must be objects",
10628 );
10629 }
10630 assert_hello_drops_role_declarations_with_warning(
10631 json!([]),
10632 "role_declarations must be an object",
10633 );
10634 }
10635
10636 #[test]
10637 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
10638 assert_hello_drops_role_declarations_with_warning(
10639 json!({"routing/v1": {}, "bad key": {}}),
10640 "role_declarations key must use capability-name grammar",
10641 );
10642 }
10643
10644 #[test]
10645 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
10646 assert_hello_drops_role_declarations_with_warning(
10647 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
10648 "role_declarations exceeds 16384 compact JSON bytes",
10649 );
10650 }
10651
10652 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
10653 let mut body = serde_json::to_value(ModuleHelloBody {
10654 manifest: manifest("flow-ops", PROTOCOL_VERSION),
10655 protocol_ver: PROTOCOL_VERSION,
10656 control_ops: None,
10657 launch_nonce: None,
10658 })
10659 .unwrap();
10660 body["manifest"]["provides"] = json!([{
10661 "role": "management_surface",
10662 "operations": [
10663 {"name": "diagnostics.findings", "kind": "query"},
10664 {"name": "reset", "kind": "mutate"}
10665 ],
10666 "config_schema": {}, "observability": [], "identity_scope": []
10667 }]);
10668 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
10669 if let Some(flow_operations) = flow_operations {
10670 body["manifest"]["flow_operations"] = flow_operations;
10671 }
10672 Frame::build(
10673 FrameType::Hello,
10674 control_flags(),
10675 0,
10676 0,
10677 401,
10678 serde_json::to_vec(&body).unwrap(),
10679 )
10680 .unwrap()
10681 }
10682
10683 #[test]
10684 fn catalog_list_mirrors_flow_operations_unchanged() {
10685 for (capability, flow_operations) in [
10688 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
10689 ("call-modes/v1", json!(["diagnostics.findings"])),
10690 ("flow-scopes/v1", json!([])),
10691 ] {
10692 let registry = Arc::new(Registry::default());
10693 let handler = ControlHandler::new(Arc::clone(®istry));
10694 let connection = ConnectionId::new(401);
10695 let response = handler
10696 .handle_control(
10697 connection,
10698 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
10699 )
10700 .unwrap();
10701 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10702 let registered = registry
10703 .get_module_by_connection(connection)
10704 .unwrap()
10705 .unwrap()
10706 .manifest;
10707 assert_eq!(
10708 serde_json::to_value(®istered.flow_operations).unwrap(),
10709 flow_operations
10710 );
10711 let (raw, entry) = role_declarations_catalog(&handler);
10712 assert_eq!(raw["flow_operations"], flow_operations);
10713 assert_eq!(entry.flow_operations, registered.flow_operations);
10714 }
10715
10716 let registry = Arc::new(Registry::default());
10717 let handler = ControlHandler::new(Arc::clone(®istry));
10718 let response = handler
10719 .handle_control(
10720 ConnectionId::new(401),
10721 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
10722 )
10723 .unwrap();
10724 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10725 let (raw, entry) = role_declarations_catalog(&handler);
10726 assert!(raw.get("flow_operations").is_none());
10727 assert_eq!(entry.flow_operations, None);
10728 }
10729
10730 #[test]
10731 fn operation_deferral_hello_accepts_valid_and_invalid_maps_and_catalog_relays_only_valid() {
10732 let oversized: serde_json::Map<String, Value> = (0..257)
10733 .map(|n| (format!("tool:t{n}"), json!(true)))
10734 .collect();
10735 for (capabilities, map, valid) in [
10736 (
10737 json!(["scope-federated/v1"]),
10738 json!({"tool:read":true,"management:reset":true}),
10739 true,
10740 ),
10741 (json!(["scope-federated/v1"]), json!({}), true),
10742 (json!([]), json!({"tool:read":true}), false),
10743 (
10744 json!(["scope-federated/v1"]),
10745 json!({"tool:valid":true,"tool:bad":false}),
10746 false,
10747 ),
10748 (
10749 json!(["scope-federated/v1"]),
10750 json!({"tool:valid":true,"management:missing":true}),
10751 false,
10752 ),
10753 (
10754 json!(["scope-federated/v1"]),
10755 Value::Object(oversized),
10756 false,
10757 ),
10758 ] {
10759 let capture = EventCapture::default();
10760 let _subscriber = tracing::subscriber::set_default(
10761 tracing_subscriber::registry().with(capture.clone()),
10762 );
10763 let registry = Arc::new(Registry::default());
10764 let handler = ControlHandler::new(Arc::clone(®istry));
10765 let hello = hello_with_flow_operations(capabilities, None);
10766 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
10767 body["manifest"]["operation_deferral"] = map.clone();
10768 let hello = Frame::build(
10769 FrameType::Hello,
10770 control_flags(),
10771 0,
10772 0,
10773 401,
10774 serde_json::to_vec(&body).unwrap(),
10775 )
10776 .unwrap();
10777 let response = handler
10778 .handle_control(ConnectionId::new(401), hello)
10779 .unwrap();
10780 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10781 let (raw, entry) = role_declarations_catalog(&handler);
10782 if valid {
10783 assert_eq!(raw["operation_deferral"], map);
10784 assert_eq!(
10785 serde_json::to_value(entry.operation_deferral.unwrap()).unwrap(),
10786 map
10787 );
10788 } else {
10789 assert!(entry.operation_deferral.is_none());
10790 assert!(raw.get("operation_deferral").is_none());
10791 assert_eq!(
10792 capture
10793 .events()
10794 .iter()
10795 .filter(|event| event.fields.get("message").map(String::as_str)
10796 == Some("dropping operation_deferral from HELLO"))
10797 .count(),
10798 1
10799 );
10800 }
10801 }
10802 }
10803
10804 fn assert_hello_drops_flow_operations_with_warning(
10805 capabilities: Value,
10806 flow_operations: Value,
10807 reason: &str,
10808 ) {
10809 let registry = Arc::new(Registry::default());
10810 let handler = ControlHandler::new(Arc::clone(®istry));
10811 let capture = EventCapture::default();
10812 let _subscriber =
10813 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10814 let response = handler
10815 .handle_control(
10816 ConnectionId::new(401),
10817 hello_with_flow_operations(capabilities, Some(flow_operations)),
10818 )
10819 .unwrap();
10820 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10821 let registered = registry
10822 .get_module_by_connection(ConnectionId::new(401))
10823 .unwrap()
10824 .expect("the module still registers")
10825 .manifest;
10826 assert_eq!(registered.module_id, "flow-ops");
10827 assert_eq!(registered.flow_operations, None);
10828 let (raw, entry) = role_declarations_catalog(&handler);
10829 assert_eq!(raw["module_id"], "flow-ops");
10830 assert!(raw.get("flow_operations").is_none());
10831 assert_eq!(entry.flow_operations, None);
10832 let warnings: Vec<_> = capture
10833 .events()
10834 .into_iter()
10835 .filter(|event| {
10836 event.fields.get("message").map(String::as_str)
10837 == Some("dropping flow_operations from HELLO")
10838 })
10839 .collect();
10840 assert_eq!(warnings.len(), 1);
10841 assert_eq!(warnings[0].level, tracing::Level::WARN);
10842 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
10843 assert_eq!(warnings[0].fields["reason"], reason);
10844 }
10845
10846 #[test]
10847 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
10848 assert_hello_drops_flow_operations_with_warning(
10849 json!(["flow-scopes/v1"]),
10850 json!(["diagnostics.findings", "missing"]),
10851 "flow_operations names 'missing', which is not a declared management operation",
10852 );
10853 }
10854
10855 #[test]
10856 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
10857 assert_hello_drops_flow_operations_with_warning(
10858 json!([]),
10859 json!(["diagnostics.findings"]),
10860 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10861 );
10862 assert_hello_drops_flow_operations_with_warning(
10864 json!(["agent-run-scopes/v1"]),
10865 json!([]),
10866 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10867 );
10868 }
10869
10870 #[test]
10871 fn hello_malformed_flow_operations_is_dropped_with_warning() {
10872 assert_hello_drops_flow_operations_with_warning(
10873 json!(["flow-scopes/v1"]),
10874 json!(["reset", "reset"]),
10875 "flow_operations names 'reset' more than once",
10876 );
10877 assert_hello_drops_flow_operations_with_warning(
10878 json!(["flow-scopes/v1"]),
10879 json!(vec!["reset"; 257]),
10880 "flow_operations lists 257 names, more than 256",
10881 );
10882 assert_hello_drops_flow_operations_with_warning(
10883 json!(["flow-scopes/v1"]),
10884 json!("reset"),
10885 "flow_operations must be an array of operation names",
10886 );
10887 }
10888
10889 #[test]
10890 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
10891 let registry = Arc::new(Registry::default());
10892 let handler = ControlHandler::new(registry.clone());
10893 let descriptor = json!({
10894 "schema": "ckios.richui.manifest/1",
10895 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10896 "bytes": 128,
10897 "chunks": 2
10898 });
10899 let malformed = [
10900 ("schema", json!("")),
10901 ("schema", json!("a".repeat(65))),
10902 ("schema", json!("Bad schema")),
10903 (
10904 "manifest_digest",
10905 json!(format!("sha256:{}", "A".repeat(64))),
10906 ),
10907 ("manifest_digest", json!("sha256:abc")),
10908 ("bytes", json!(0)),
10909 ("bytes", json!(16_777_217)),
10910 ("bytes", json!("128")),
10911 ("chunks", json!(0)),
10912 ("chunks", json!(1025)),
10913 ("chunks", json!("2")),
10914 ];
10915 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
10916 let mut invalid = descriptor.clone();
10917 invalid[field] = invalid_value;
10918 let response = handler
10919 .handle_control(
10920 ConnectionId::new(210 + index as u64),
10921 hello_frame_with_rich_ui(
10922 &format!("bad-rich-ui-{index}"),
10923 PROTOCOL_VERSION,
10924 210 + index as u64,
10925 None,
10926 invalid,
10927 ),
10928 )
10929 .unwrap();
10930 assert_eq!(response[0].header.ty, FrameType::Error);
10931 let error = parse_error(&response[0]);
10932 assert_eq!(error["code"], "invalid_manifest");
10933 assert!(error["message"]
10934 .as_str()
10935 .unwrap()
10936 .contains(&format!("rich_ui.{field}")));
10937 assert!(registry
10938 .get_module(&format!("bad-rich-ui-{index}"))
10939 .unwrap()
10940 .is_none());
10941 }
10942 }
10943
10944 #[test]
10945 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
10946 let registry = Arc::new(Registry::default());
10947 let handler = ControlHandler::new(Arc::clone(®istry));
10948 let connection = ConnectionId::new(219);
10949 registry
10950 .register_with_control_ops(
10951 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
10952 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
10953 "bad schema",
10954 format!("sha256:{}", "a".repeat(64)),
10955 128,
10956 2,
10957 )))
10958 .build(),
10959 PROTOCOL_VERSION,
10960 connection,
10961 Vec::new(),
10962 )
10963 .unwrap();
10964 let request =
10965 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
10966 let response = handler
10967 .handle_catalog_update(connection, request, Vec::new(), None, None)
10968 .unwrap();
10969 assert_eq!(response[0].header.ty, FrameType::Error);
10970 let error = parse_error(&response[0]);
10971 assert_eq!(error["code"], "invalid_manifest");
10972 assert!(error["message"]
10973 .as_str()
10974 .unwrap()
10975 .contains("rich_ui.schema"));
10976 }
10977
10978 #[test]
10979 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
10980 let registry = Arc::new(Registry::default());
10981 let supervisor = SupervisorHandle::new();
10982 supervisor.open_swap("provider", "candidate-nonce".to_string());
10983 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10984 let connection = ConnectionId::new(218);
10985 let hello = handler
10986 .handle_control(
10987 connection,
10988 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
10989 )
10990 .unwrap();
10991 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10992 assert!(registry.list_modules().unwrap().1.is_empty());
10993
10994 registry.promote_candidate("provider").unwrap().unwrap();
10995 let request = Frame::build(
10996 FrameType::Request,
10997 control_flags(),
10998 0,
10999 0,
11000 219,
11001 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
11002 )
11003 .unwrap();
11004 let response = handler.handle_catalog_list(request, None).unwrap();
11005 let ClientControlResponse::CatalogList { modules, .. } =
11006 serde_json::from_slice(&response[0].body).unwrap()
11007 else {
11008 panic!("expected catalog.list");
11009 };
11010 assert_eq!(modules.len(), 1);
11011 assert!(modules[0].nonce_verified);
11012 }
11013
11014 #[test]
11015 fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
11016 for provenance_json in [
11017 r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
11018 r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
11019 ] {
11020 let registry = Arc::new(Registry::default());
11021 let handler = ControlHandler::new(registry);
11022 let hello = Frame::build(
11023 FrameType::Hello,
11024 control_flags(),
11025 0,
11026 0,
11027 220,
11028 serde_json::to_vec(&ModuleHelloBody {
11029 manifest: ModuleManifest::builder("built", "0.1.0")
11030 .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
11031 .build(),
11032 protocol_ver: PROTOCOL_VERSION,
11033 control_ops: None,
11034 launch_nonce: None,
11035 })
11036 .unwrap(),
11037 )
11038 .unwrap();
11039 assert_eq!(
11040 handler
11041 .handle_control(ConnectionId::new(220), hello)
11042 .unwrap()[0]
11043 .header
11044 .ty,
11045 FrameType::HelloAck
11046 );
11047 let request = Frame::build(
11048 FrameType::Request,
11049 control_flags(),
11050 0,
11051 0,
11052 221,
11053 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
11054 )
11055 .unwrap();
11056 let response = handler.handle_catalog_list(request, None).unwrap();
11057 let ClientControlResponse::CatalogList { modules, .. } =
11058 serde_json::from_slice(&response[0].body).unwrap()
11059 else {
11060 panic!("expected catalog.list");
11061 };
11062 assert_eq!(modules.len(), 1);
11063 assert_eq!(
11064 serde_json::to_vec(&modules[0]).unwrap(),
11065 format!(
11066 r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
11067 )
11068 .as_bytes()
11069 );
11070 }
11071 }
11072
11073 #[test]
11074 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
11075 let registry = Arc::new(Registry::default());
11076 let handler = ControlHandler::new(Arc::clone(®istry));
11077 let hello = Frame::build(
11078 FrameType::Hello,
11079 control_flags(),
11080 0,
11081 0,
11082 220,
11083 serde_json::to_vec(&ModuleHelloBody {
11084 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
11085 protocol_ver: PROTOCOL_VERSION,
11086 control_ops: None,
11087 launch_nonce: Some("unverified".to_string()),
11088 })
11089 .unwrap(),
11090 )
11091 .unwrap();
11092 assert_eq!(
11093 handler
11094 .handle_control(ConnectionId::new(220), hello)
11095 .unwrap()[0]
11096 .header
11097 .ty,
11098 FrameType::HelloAck
11099 );
11100 let request = Frame::build(
11101 FrameType::Request,
11102 control_flags(),
11103 0,
11104 0,
11105 221,
11106 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
11107 )
11108 .unwrap();
11109 let response = handler.handle_catalog_list(request, None).unwrap();
11110 let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
11111 assert!(wire["modules"][0].get("provenance").is_none());
11112 let ClientControlResponse::CatalogList { modules, .. } =
11113 serde_json::from_slice(&response[0].body).unwrap()
11114 else {
11115 panic!("expected catalog.list");
11116 };
11117 assert_eq!(
11118 serde_json::to_vec(&modules[0]).unwrap(),
11119 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
11120 );
11121 assert!(!modules[0].nonce_verified);
11122 assert!(modules[0].provenance.is_none());
11123 }
11124
11125 #[test]
11126 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
11127 let registry = Arc::new(Registry::default());
11128 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
11129 [("vault".to_string(), true), ("squatter".to_string(), true)],
11130 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
11131 );
11132 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
11133 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
11134 provides: vec!["credentials-provider/v1".to_string()],
11135 requires: Vec::new(),
11136 must_never_reach: Vec::new(),
11137 });
11138 let frame = Frame::build(
11139 FrameType::Hello,
11140 control_flags(),
11141 0,
11142 0,
11143 77,
11144 serde_json::to_vec(&ModuleHelloBody {
11145 manifest: squatter,
11146 protocol_ver: PROTOCOL_VERSION,
11147 control_ops: None,
11148 launch_nonce: None,
11149 })
11150 .expect("HELLO serializes"),
11151 )
11152 .expect("HELLO frame builds");
11153 let response = handler
11154 .handle_control(ConnectionId::new(77), frame)
11155 .expect("reserved claim receives a typed refusal");
11156 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
11157 assert_eq!(
11158 registry
11159 .active_registration_count()
11160 .expect("registry reads"),
11161 0,
11162 "a reserved capability refusal must not leave a catalog entry"
11163 );
11164 }
11165
11166 #[test]
11167 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
11168 for settlement in ["timeout", "inconclusive", "drop"] {
11169 let breakers = RouteBindBreakers::default();
11170 let RouteBindAdmission::Admitted {
11171 guard: mut old,
11172 probe: false,
11173 } = breakers.admit("prov")
11174 else {
11175 panic!("ordinary relay admitted")
11176 };
11177 let RouteBindAdmission::Admitted {
11178 guard: mut opener, ..
11179 } = breakers.admit("prov")
11180 else {
11181 panic!("second relay admitted")
11182 };
11183 assert!(
11184 !opener
11185 .record_timeout(1, Duration::ZERO)
11186 .unwrap()
11187 .reopened_after_probe
11188 );
11189 let RouteBindAdmission::Admitted {
11190 guard: mut probe,
11191 probe: true,
11192 } = breakers.admit("prov")
11193 else {
11194 panic!("one cooldown probe admitted")
11195 };
11196 match settlement {
11197 "timeout" => assert!(
11198 !old.record_timeout(1, Duration::ZERO)
11199 .unwrap()
11200 .reopened_after_probe
11201 ),
11202 "inconclusive" => old.record_inconclusive(),
11203 "drop" => drop(old),
11204 _ => unreachable!(),
11205 }
11206 assert!(
11207 matches!(
11208 breakers.admit("prov"),
11209 RouteBindAdmission::Refused {
11210 probe_in_flight: true,
11211 ..
11212 }
11213 ),
11214 "{settlement} of a pre-open relay cannot release the real probe"
11215 );
11216 assert!(
11217 probe
11218 .record_timeout(1, Duration::ZERO)
11219 .unwrap()
11220 .reopened_after_probe
11221 );
11222 assert!(matches!(
11223 breakers.admit("prov"),
11224 RouteBindAdmission::Admitted { probe: true, .. }
11225 ));
11226 }
11227 let breakers = RouteBindBreakers::default();
11228 let admit = || match breakers.admit("prov") {
11229 RouteBindAdmission::Admitted { guard, .. } => guard,
11230 _ => panic!("relay admitted"),
11231 };
11232 admit().record_timeout(1, Duration::ZERO);
11233 let mut old_probe = admit();
11234 breakers.reset_for_new_module_connection("prov");
11235 admit().record_timeout(1, Duration::ZERO);
11236 let _new_probe = admit();
11237 old_probe.record_inconclusive();
11238 assert!(matches!(
11239 breakers.admit("prov"),
11240 RouteBindAdmission::Refused {
11241 probe_in_flight: true,
11242 ..
11243 }
11244 ));
11245 }
11246
11247 #[tokio::test]
11248 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
11249 for candidate in [false, true] {
11250 let registry = Arc::new(Registry::default());
11251 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
11252 [("vault".to_string(), true), ("squatter".to_string(), true)],
11253 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
11254 );
11255 let conn = ConnectionId::new(77);
11256 let (ctx, mut rx) = route_ctx(conn);
11257 let initial = capability_manifest("squatter", &[], &[]);
11258 if candidate {
11259 registry
11260 .register_candidate_with_control_ops(
11261 initial.clone(),
11262 PROTOCOL_VERSION,
11263 conn,
11264 module_baseline_control_ops(),
11265 )
11266 .unwrap();
11267 handler
11268 .forwarding
11269 .register_candidate_module_connection(
11270 conn,
11271 "squatter".to_string(),
11272 PROTOCOL_VERSION,
11273 manifest_concurrency(&initial),
11274 ctx.egress.clone(),
11275 )
11276 .unwrap();
11277 } else {
11278 hello_via_sink(
11279 &handler,
11280 &ctx,
11281 &mut rx,
11282 hello_frame_with_manifest(initial.clone(), 1),
11283 )
11284 .await;
11285 }
11286 let response = handler
11287 .handle_control_frame(
11288 &ctx,
11289 catalog_update_with_capabilities_frame(
11290 2,
11291 capability_manifest("squatter", &["credentials-provider/v1"], &[])
11292 .capabilities
11293 .unwrap(),
11294 ),
11295 )
11296 .await
11297 .unwrap();
11298 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
11299 assert_eq!(
11300 registry
11301 .get_module_by_connection(conn)
11302 .unwrap()
11303 .unwrap()
11304 .manifest,
11305 initial
11306 );
11307 }
11308 }
11309
11310 #[test]
11311 fn server_describe_surfaces_required_capability_verdict_fields() {
11312 let registry = Arc::new(Registry::default());
11313 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
11314 [
11315 ("consumer".to_string(), true),
11316 ("provider".to_string(), false),
11317 ],
11318 BTreeMap::new(),
11319 );
11320 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
11321 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
11322 provides: Vec::new(),
11323 requires: vec![subc_protocol::manifest::CapabilityRequirement {
11324 capability: "credentials-provider/v1".to_string(),
11325 need: subc_protocol::manifest::CapabilityNeed::Required,
11326 }],
11327 must_never_reach: Vec::new(),
11328 });
11329 let hello = Frame::build(
11330 FrameType::Hello,
11331 control_flags(),
11332 0,
11333 0,
11334 78,
11335 serde_json::to_vec(&ModuleHelloBody {
11336 manifest: consumer,
11337 protocol_ver: PROTOCOL_VERSION,
11338 control_ops: None,
11339 launch_nonce: None,
11340 })
11341 .expect("HELLO serializes"),
11342 )
11343 .expect("HELLO frame builds");
11344 handler
11345 .handle_control(ConnectionId::new(78), hello)
11346 .expect("consumer registers");
11347 let describe = Frame::build(
11348 FrameType::Request,
11349 control_flags(),
11350 0,
11351 0,
11352 79,
11353 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
11354 .expect("request serializes"),
11355 )
11356 .expect("describe frame builds");
11357 let response = handler
11358 .handle_server_describe(describe)
11359 .expect("server.describe succeeds");
11360 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
11361 let requirement = &rendered["capability_requirements"][0];
11362 assert_eq!(requirement["consumer"], "consumer");
11363 assert_eq!(requirement["verdict"], "never_provided");
11364 assert_eq!(requirement["episode_seq"], 1);
11365 assert_eq!(requirement["config_satisfiable"], false);
11366 assert_eq!(requirement["runtime_available"], false);
11367 assert!(requirement["detail"]
11368 .as_str()
11369 .expect("detail string")
11370 .contains("credentials-provider/v1"));
11371 }
11372
11373 fn deregister_required_provider(
11379 daemon_shutdown_begun: bool,
11380 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
11381 const CAPABILITY: &str = "project-identity/v1";
11382 let registry = Arc::new(Registry::default());
11383 let daemon_shutdown = DaemonShutdownFlag::default();
11384 let handler = ControlHandler::new(Arc::clone(®istry))
11385 .with_capability_config(
11386 [
11387 ("consumer".to_string(), true),
11388 ("provider".to_string(), true),
11389 ],
11390 BTreeMap::new(),
11391 )
11392 .with_daemon_shutdown_flag(daemon_shutdown.clone());
11393 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
11394 consumer.capabilities = Some(CapabilityDeclarations {
11395 provides: Vec::new(),
11396 requires: vec![subc_protocol::manifest::CapabilityRequirement {
11397 capability: CAPABILITY.to_string(),
11398 need: subc_protocol::manifest::CapabilityNeed::Required,
11399 }],
11400 must_never_reach: Vec::new(),
11401 });
11402 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
11403 let provider_conn = ConnectionId::new(182);
11404 for (conn, manifest, corr) in [
11405 (ConnectionId::new(181), consumer, 181),
11406 (provider_conn, provider, 182),
11407 ] {
11408 let hello = handler
11409 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
11410 .expect("module registers");
11411 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11412 }
11413 assert_eq!(
11414 handler.capability_evaluator.verdict("consumer", CAPABILITY),
11415 Some(CapabilityVerdict::Provided),
11416 "the registered provider must satisfy the requirement first"
11417 );
11418 if daemon_shutdown_begun {
11419 daemon_shutdown.set_for_test();
11420 }
11421
11422 let capture = EventCapture::default();
11423 let _subscriber =
11424 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11425 handler
11426 .cleanup_connection(provider_conn)
11427 .expect("provider deregisters");
11428 let events = capture
11429 .events()
11430 .into_iter()
11431 .filter(|event| {
11432 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
11433 })
11434 .collect();
11435 (
11436 events,
11437 handler.capability_evaluator.verdict("consumer", CAPABILITY),
11438 )
11439 }
11440
11441 #[test]
11442 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
11443 let (events, verdict) = deregister_required_provider(true);
11444 assert_eq!(
11445 verdict,
11446 Some(CapabilityVerdict::NeverProvided),
11447 "shutdown must not change the verdict route.open refusals read"
11448 );
11449 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11450 let event = &events[0];
11451 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
11452 assert_eq!(
11453 event.fields.get("cause").map(String::as_str),
11454 Some("\"daemon_shutdown\""),
11455 "{event:?}"
11456 );
11457 assert_eq!(
11458 event.fields.get("verdict").map(String::as_str),
11459 Some("\"never_provided\""),
11460 "{event:?}"
11461 );
11462 }
11463
11464 #[test]
11465 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
11466 let (events, verdict) = deregister_required_provider(false);
11467 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
11468 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11469 let event = &events[0];
11470 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
11471 assert_eq!(event.fields.get("cause"), None, "{event:?}");
11472 assert_eq!(
11473 event.fields.get("verdict").map(String::as_str),
11474 Some("\"never_provided\""),
11475 "{event:?}"
11476 );
11477 }
11478
11479 #[test]
11480 fn catalog_list_omits_capabilities_for_legacy_manifest() {
11481 let registry = Arc::new(Registry::default());
11482 let handler = ControlHandler::new(Arc::clone(®istry));
11483 let hello = handler
11484 .handle_control(
11485 ConnectionId::new(101),
11486 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
11487 )
11488 .expect("legacy HELLO registers");
11489 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11490
11491 let request = Frame::build(
11492 FrameType::Request,
11493 control_flags(),
11494 0,
11495 0,
11496 102,
11497 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
11498 .expect("catalog request serializes"),
11499 )
11500 .expect("catalog request frame builds");
11501 let response = handler
11502 .handle_catalog_list(request, None)
11503 .expect("catalog list succeeds");
11504 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
11505 assert!(
11506 body["modules"][0].get("capabilities").is_none(),
11507 "legacy manifest must retain an absent capabilities field on catalog.list"
11508 );
11509 }
11510
11511 #[test]
11512 fn hello_ack_omits_storage_when_no_storage_config() {
11513 let registry = Arc::new(Registry::default());
11514 let handler = ControlHandler::new(Arc::clone(®istry));
11515 let responses = handler
11516 .handle_control(
11517 ConnectionId::new(1),
11518 hello_frame("aft", PROTOCOL_VERSION, 7),
11519 )
11520 .unwrap();
11521 let ack = parse_ack(&responses[0]);
11522 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
11523 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
11524 }
11525
11526 #[tokio::test]
11527 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
11528 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
11529 let registry = Arc::new(Registry::default());
11530 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
11531 let responses = handler
11532 .handle_control(
11533 ConnectionId::new(1),
11534 hello_frame("aft", PROTOCOL_VERSION, 7),
11535 )
11536 .unwrap();
11537 let ack = parse_ack(&responses[0]);
11538 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
11539
11540 let described = handler
11541 .handle_control_frame(
11542 &route_ctx(ConnectionId::new(2)).0,
11543 Frame::build(
11544 FrameType::Request,
11545 control_flags(),
11546 0,
11547 0,
11548 9,
11549 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11550 )
11551 .unwrap(),
11552 )
11553 .await
11554 .unwrap();
11555 let ClientControlResponse::ServerDescribe { machine_id, .. } =
11556 serde_json::from_slice(&described[0].body).unwrap()
11557 else {
11558 panic!("server.describe answered with another shape");
11559 };
11560 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
11561 }
11562
11563 #[test]
11564 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
11565 let registry = Arc::new(Registry::default());
11568 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
11569 crate::daemon_config::StorageConfig::Sqlite {
11570 data_home: std::path::PathBuf::from("/data"),
11571 },
11572 ));
11573
11574 let responses = handler
11575 .handle_control(
11576 ConnectionId::new(1),
11577 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
11578 )
11579 .unwrap();
11580 let ack = parse_ack(&responses[0]);
11581 assert_eq!(
11582 ack.storage,
11583 Some(serde_json::json!({
11584 "module_id": "alfonso-routing",
11585 "storage_namespace": "default",
11586 "isolation": { "kind": "module" },
11587 "backend": {
11588 "backend": "sqlite",
11589 "path": "/data/cortexkit/alfonso-routing/store.db"
11590 }
11591 })),
11592 "the delivered descriptor is the module's own sqlite store path"
11593 );
11594 }
11595
11596 #[test]
11597 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
11598 let registry = Arc::new(Registry::default());
11599 let handler = ControlHandler::new(Arc::clone(®istry));
11600 let conn = ConnectionId::new(1);
11601 let responses = handler
11602 .handle_control(
11603 conn,
11604 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11605 )
11606 .unwrap();
11607 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11608 let registration = registry.get_module("aft").unwrap().unwrap();
11609 assert_eq!(registration.control_ops, module_baseline_control_ops());
11610
11611 let frame =
11612 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
11613 assert!(handler
11614 .guard_module_control_op(&frame, "aft", "route.bind")
11615 .unwrap()
11616 .is_none());
11617 let error = handler
11618 .guard_module_control_op(&frame, "aft", "test.synthetic")
11619 .unwrap()
11620 .expect("synthetic ungranted op should be rejected");
11621 assert_eq!(error.header.ty, FrameType::Error);
11622 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
11623 }
11624
11625 #[test]
11626 fn hello_control_ops_some_adds_optional_grants() {
11627 let registry = Arc::new(Registry::default());
11628 let handler = ControlHandler::new(Arc::clone(®istry));
11629 handler
11630 .handle_control(
11631 ConnectionId::new(1),
11632 hello_frame_with_control_ops(
11633 "aft",
11634 PROTOCOL_VERSION,
11635 7,
11636 Some(vec![
11637 "future.synthetic".to_string(),
11638 "route.bind".to_string(),
11639 ]),
11640 ),
11641 )
11642 .unwrap();
11643 let registration = registry.get_module("aft").unwrap().unwrap();
11644 assert_eq!(
11645 registration.control_ops,
11646 vec![
11647 "route.bind".to_string(),
11648 "route.status".to_string(),
11649 "future.synthetic".to_string(),
11650 ]
11651 );
11652 let frame =
11653 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
11654 assert!(handler
11655 .guard_module_control_op(&frame, "aft", "future.synthetic")
11656 .unwrap()
11657 .is_none());
11658 }
11659
11660 #[tokio::test]
11661 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
11662 let registry = Arc::new(Registry::default());
11663 let forwarding = Arc::new(ForwardingTable::default());
11664 let handler =
11665 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11666 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
11667 hello_via_sink(
11668 &handler,
11669 &module_ctx,
11670 &mut module_rx,
11671 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11672 )
11673 .await;
11674
11675 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
11676 let responses = handler
11677 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
11678 .await
11679 .unwrap();
11680 assert_eq!(responses.len(), 1);
11681 assert_eq!(responses[0].header.ty, FrameType::Error);
11682 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
11683 assert!(module_rx.try_recv().is_err());
11684 }
11685
11686 #[tokio::test]
11687 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
11688 let registry = Arc::new(Registry::default());
11689 let forwarding = Arc::new(ForwardingTable::default());
11690 let handler =
11691 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11692 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
11693 hello_via_sink(
11694 &handler,
11695 &module_ctx,
11696 &mut module_rx,
11697 hello_frame_with_control_ops(
11698 "aft",
11699 PROTOCOL_VERSION,
11700 7,
11701 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11702 ),
11703 )
11704 .await;
11705
11706 let project_root = unique_project_root("demux");
11707 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
11708 let route_handler = handler.clone();
11709 let route_task = tokio::spawn(async move {
11710 route_handler
11711 .handle_control_frame(
11712 &route_client_ctx,
11713 route_open_frame(100, "aft", project_root),
11714 )
11715 .await
11716 .unwrap()
11717 });
11718 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11719 .await
11720 .unwrap()
11721 .unwrap();
11722 assert!(matches!(
11723 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
11724 ModuleControlRequest::RouteBind { .. }
11725 ));
11726
11727 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
11728 let health_handler = handler.clone();
11729 let health_task = tokio::spawn(async move {
11730 health_handler
11731 .handle_control_frame(
11732 &health_client_ctx,
11733 supervisor_health_probe_frame(101, "aft"),
11734 )
11735 .await
11736 .unwrap()
11737 });
11738 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11739 .await
11740 .unwrap()
11741 .unwrap();
11742 assert_eq!(
11743 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11744 ModuleControlRequest::HealthCheck {}
11745 );
11746
11747 handler
11748 .handle_control_frame(
11749 &module_ctx,
11750 health_response(health_frame.header.corr, HealthStatus::Degraded),
11751 )
11752 .await
11753 .unwrap();
11754 let health_response = health_task.await.unwrap();
11755 assert_eq!(health_response.len(), 1);
11756 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11757 ClientControlResponse::SupervisorHealthProbe {
11758 module_id,
11759 status,
11760 detail,
11761 metrics,
11762 } => {
11763 assert_eq!(module_id, "aft");
11764 assert_eq!(status, HealthStatus::Degraded);
11765 assert_eq!(detail.as_deref(), Some("warming"));
11766 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
11767 }
11768 other => panic!("unexpected health response: {other:?}"),
11769 }
11770
11771 handler
11772 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11773 .await
11774 .unwrap();
11775 let route_response = route_task.await.unwrap();
11776 assert!(route_response.is_empty());
11777 let published = route_client_rx.recv().await.unwrap();
11778 assert!(matches!(
11779 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11780 ClientControlResponse::RouteOpen { .. }
11781 ));
11782 }
11783
11784 async fn relay_route_open(
11789 handler: &ControlHandler,
11790 client_connection: ConnectionId,
11791 client_egress: &FrameSink,
11792 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11793 corr: u64,
11794 module_id: &str,
11795 project_root_label: &str,
11796 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
11797 let ctx = RouteCtx {
11798 connection_id: client_connection,
11799 egress: client_egress.clone(),
11800 };
11801 let handler = handler.clone();
11802 let project_root = unique_project_root(project_root_label);
11803 let module_id = module_id.to_string();
11804 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
11805 let task = tokio::spawn(async move {
11806 let _guard = tracing::dispatcher::set_default(&dispatch);
11807 handler
11808 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
11809 .await
11810 .unwrap()
11811 });
11812 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
11813 .await
11814 .expect("module receives the relayed route.bind")
11815 .expect("module egress is open");
11816 (task, bind.frame)
11817 }
11818
11819 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
11820 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
11821 ModuleControlRequest::RouteBind {
11822 route_channel,
11823 epoch,
11824 ..
11825 } => (route_channel, epoch),
11826 other => panic!("expected a route.bind request, got {other:?}"),
11827 }
11828 }
11829
11830 fn published_route(frame: &Frame) -> (u16, u32) {
11831 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
11832 ClientControlResponse::RouteOpen {
11833 route_channel,
11834 route_epoch,
11835 ..
11836 } => (route_channel, route_epoch),
11837 other => panic!("expected a route.open response, got {other:?}"),
11838 }
11839 }
11840
11841 #[tokio::test]
11861 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
11862 let registry = Arc::new(Registry::default());
11863 let forwarding = Arc::new(ForwardingTable::default());
11864 let handler =
11865 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11866
11867 let module_connection = ConnectionId::new(30);
11868 let (module_ctx, mut module_rx) = route_ctx(module_connection);
11869 hello_via_sink(
11870 &handler,
11871 &module_ctx,
11872 &mut module_rx,
11873 hello_frame("aft", PROTOCOL_VERSION, 7),
11874 )
11875 .await;
11876
11877 let dying_client = ConnectionId::new(31);
11878 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
11879
11880 let (first_task, first_bind) = relay_route_open(
11883 &handler,
11884 dying_client,
11885 &dying_ctx.egress,
11886 &mut module_rx,
11887 100,
11888 "aft",
11889 "closing-first",
11890 )
11891 .await;
11892 handler
11893 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
11894 .await
11895 .unwrap();
11896 assert!(first_task.await.unwrap().is_empty());
11897 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
11898
11899 let (second_task, second_bind) = relay_route_open(
11901 &handler,
11902 dying_client,
11903 &dying_ctx.egress,
11904 &mut module_rx,
11905 101,
11906 "aft",
11907 "closing-second",
11908 )
11909 .await;
11910 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
11911
11912 assert!(forwarding
11915 .escalate_client_delivery_failure(
11916 dying_client,
11917 first_channel,
11918 first_epoch,
11919 CloseReason::new(
11920 "module_to_client_delivery_failed",
11921 "client egress refused a module frame",
11922 ),
11923 crate::forwarding::UndeliveredFrame {
11924 module_id: None,
11925 sink: &dying_ctx.egress,
11926 },
11927 )
11928 .unwrap());
11929 assert!(!dying_ctx.egress.is_closed());
11930
11931 let ack = handler
11933 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
11934 .await;
11935 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11936 if module_loop_error.is_some() {
11937 handler.cleanup_connection(module_connection).unwrap();
11941 }
11942 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11946 .await
11947 .ok()
11948 .flatten();
11949
11950 assert!(
11952 registry
11953 .get_module_by_connection(module_connection)
11954 .unwrap()
11955 .is_some(),
11956 "one client's closing connection ended the shared module connection: \
11957 {module_loop_error:?}"
11958 );
11959 let cotenant = ConnectionId::new(32);
11961 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11962 let (cotenant_task, cotenant_bind) = relay_route_open(
11963 &handler,
11964 cotenant,
11965 &cotenant_ctx.egress,
11966 &mut module_rx,
11967 102,
11968 "aft",
11969 "closing-cotenant",
11970 )
11971 .await;
11972 handler
11973 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
11974 .await
11975 .unwrap();
11976 assert!(cotenant_task.await.unwrap().is_empty());
11977 let (cotenant_channel, cotenant_epoch) =
11978 published_route(&cotenant_rx.recv().await.unwrap());
11979 assert!(matches!(
11980 forwarding
11981 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11982 .unwrap(),
11983 DataRoute::Client(DataRouteState::Bound(_))
11984 ));
11985
11986 let goodbye = post_ack_module_frame
11989 .expect("module receives a GOODBYE for the abandoned route channel");
11990 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11991 assert_eq!(goodbye.header.channel, abandoned_channel);
11992 assert_eq!(goodbye.header.epoch, abandoned_epoch);
11993
11994 assert!(dying_rx.try_recv().is_err());
11998 let second_response = second_task.await.unwrap();
11999 assert_eq!(second_response.len(), 1);
12000 assert_eq!(
12001 parse_error(&second_response[0])["code"],
12002 "target_unavailable"
12003 );
12004 }
12005
12006 async fn catalog_snapshot(
12009 handler: &ControlHandler,
12010 client_ctx: &RouteCtx,
12011 corr: u64,
12012 ) -> (u64, Vec<(String, bool)>) {
12013 let body =
12014 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
12015 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
12016 let reply = handler
12017 .handle_control_frame(client_ctx, request)
12018 .await
12019 .unwrap()
12020 .pop()
12021 .unwrap();
12022 let ClientControlResponse::CatalogList {
12023 generation,
12024 modules,
12025 ..
12026 } = serde_json::from_slice(&reply.body).unwrap()
12027 else {
12028 panic!("not a catalog.list reply");
12029 };
12030 (
12031 generation,
12032 modules
12033 .into_iter()
12034 .map(|entry| (entry.module_id, entry.ready))
12035 .collect(),
12036 )
12037 }
12038
12039 fn hello_held_before_forwarding() -> (
12045 ControlHandler,
12046 Arc<std::sync::Barrier>,
12047 std::thread::JoinHandle<Vec<Frame>>,
12048 RouteCtx,
12049 mpsc::Receiver<crate::router::OutboundFrame>,
12050 ) {
12051 let hold = Arc::new(std::sync::Barrier::new(2));
12052 let handler = ControlHandler::with_forwarding(
12053 Arc::new(Registry::default()),
12054 Arc::new(ForwardingTable::default()),
12055 )
12056 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
12057 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
12058 let hello = {
12059 let handler = handler.clone();
12060 let module_ctx = module_ctx.clone();
12061 std::thread::spawn(move || {
12062 handler
12063 .handle_hello(
12064 module_ctx.connection_id,
12065 Some(module_ctx.egress.clone()),
12066 hello_frame("aft", PROTOCOL_VERSION, 7),
12067 )
12068 .unwrap()
12069 })
12070 };
12071 (handler, hold, hello, module_ctx, module_rx)
12072 }
12073
12074 #[tokio::test]
12083 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
12084 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
12085 let client = ConnectionId::new(41);
12086 let (client_ctx, mut client_rx) = route_ctx(client);
12087
12088 hold.wait();
12089 assert!(handler.registry.get_module("aft").unwrap().is_some());
12091 assert!(!handler
12092 .forwarding
12093 .has_live_module_connection("aft")
12094 .unwrap());
12095 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
12096 assert!(
12097 listed.is_empty(),
12098 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
12099 );
12100 let refused = handler
12101 .handle_control_frame(
12102 &client_ctx,
12103 route_open_frame(301, "aft", unique_project_root("listed-window")),
12104 )
12105 .await
12106 .unwrap();
12107 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
12108 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
12109 let duplicate = handler
12110 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
12111 .await
12112 .unwrap();
12113 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12114
12115 hold.wait();
12116 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
12117 let ack = module_rx.try_recv().unwrap().frame;
12118 assert_eq!(ack.header.ty, FrameType::HelloAck);
12119 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
12120 assert_eq!(listed, vec![("aft".to_string(), true)]);
12121
12122 let (open_task, bind) = relay_route_open(
12123 &handler,
12124 client,
12125 &client_ctx.egress,
12126 &mut module_rx,
12127 303,
12128 "aft",
12129 "listed-after",
12130 )
12131 .await;
12132 assert!(handler
12133 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12134 .await
12135 .unwrap()
12136 .is_empty());
12137 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
12138 published_route(&client_rx.try_recv().unwrap().frame);
12139 }
12140
12141 #[tokio::test]
12148 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
12149 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
12150 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12151 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
12152
12153 hold.wait();
12154 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
12155 assert!(listed.is_empty());
12156 assert_eq!(
12157 during, before,
12158 "the generation moved while the module was not listed"
12159 );
12160
12161 hold.wait();
12162 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
12163 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
12164 assert_eq!(listed, vec![("aft".to_string(), true)]);
12165 assert_eq!(
12166 after,
12167 before + 1,
12168 "the module appeared without exactly one generation bump"
12169 );
12170 }
12171
12172 #[tokio::test]
12177 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
12178 let registry = Arc::new(Registry::default());
12179 let handler = ControlHandler::with_forwarding(
12180 Arc::clone(®istry),
12181 Arc::new(ForwardingTable::default()),
12182 );
12183 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12184 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
12185 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
12186 drop(module_rx);
12187
12188 let replies = handler
12189 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
12190 .await
12191 .unwrap();
12192 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
12193 assert!(registry.get_module("aft").unwrap().is_none());
12194 assert!(registry
12195 .get_module_by_connection(module_ctx.connection_id)
12196 .unwrap()
12197 .is_none());
12198 assert_eq!(
12199 catalog_snapshot(&handler, &client_ctx, 301).await,
12200 (before, Vec::new())
12201 );
12202 }
12203
12204 async fn route_open_held_after_bind_send() -> (
12208 ControlHandler,
12209 Arc<tokio::sync::Barrier>,
12210 RouteCtx,
12211 mpsc::Receiver<crate::router::OutboundFrame>,
12212 ) {
12213 let hold = Arc::new(tokio::sync::Barrier::new(2));
12214 let handler = ControlHandler::with_forwarding(
12215 Arc::new(Registry::default()),
12216 Arc::new(ForwardingTable::default()),
12217 )
12218 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
12219 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12220 hello_via_sink(
12221 &handler,
12222 &module_ctx,
12223 &mut module_rx,
12224 hello_frame("aft", PROTOCOL_VERSION, 7),
12225 )
12226 .await;
12227 (handler, hold, module_ctx, module_rx)
12228 }
12229
12230 #[tokio::test]
12237 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
12238 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12239 let client = ConnectionId::new(41);
12240 let (client_ctx, mut client_rx) = route_ctx(client);
12241
12242 let (open_task, bind) = relay_route_open(
12243 &handler,
12244 client,
12245 &client_ctx.egress,
12246 &mut module_rx,
12247 200,
12248 "aft",
12249 "acked-before-mark",
12250 )
12251 .await;
12252 let (module_channel, module_epoch) = route_bind_channel(&bind);
12253 hold.wait().await;
12255
12256 assert!(handler
12258 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12259 .await
12260 .unwrap()
12261 .is_empty());
12262 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
12263
12264 hold.wait().await;
12265 assert!(
12266 open_task.await.unwrap().is_empty(),
12267 "route.open answers accepted"
12268 );
12269
12270 match module_rx.try_recv() {
12273 Err(mpsc::error::TryRecvError::Empty) => {}
12274 Ok(outbound) => panic!(
12275 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
12276 outbound.frame.header.ty,
12277 outbound.frame.header.channel,
12278 outbound.frame.header.epoch
12279 ),
12280 Err(err) => panic!("module egress closed: {err}"),
12281 }
12282
12283 let router = Router::with_control_handler(Arc::new(handler.clone()));
12286 let request = Frame::build(
12287 FrameType::Request,
12288 Flags::new(true, Priority::Interactive, false),
12289 client_channel,
12290 client_epoch,
12291 201,
12292 b"first request".to_vec(),
12293 )
12294 .unwrap();
12295 router
12296 .route_for_connection(&client_ctx, request)
12297 .await
12298 .unwrap();
12299 let delivered = module_rx.try_recv().unwrap().frame;
12300 assert_eq!(delivered.header.ty, FrameType::Request);
12301 assert_eq!(
12302 (delivered.header.channel, delivered.header.epoch),
12303 (module_channel, module_epoch)
12304 );
12305 assert_eq!(delivered.body, b"first request");
12306 }
12307
12308 #[tokio::test]
12314 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
12315 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12316 let client = ConnectionId::new(42);
12317 let (client_ctx, _client_rx) = route_ctx(client);
12318
12319 let (open_task, bind) = relay_route_open(
12320 &handler,
12321 client,
12322 &client_ctx.egress,
12323 &mut module_rx,
12324 300,
12325 "aft",
12326 "released-before-mark",
12327 )
12328 .await;
12329 let (module_channel, module_epoch) = route_bind_channel(&bind);
12330 hold.wait().await;
12331
12332 handler.cleanup_connection(client).unwrap();
12333 assert!(
12334 module_rx.try_recv().is_err(),
12335 "client teardown must not GOODBYE a bind route.open is still queueing"
12336 );
12337
12338 hold.wait().await;
12339 let response = open_task.await.unwrap();
12340 assert_eq!(response.len(), 1);
12341 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12342
12343 let goodbye = module_rx
12344 .try_recv()
12345 .expect("module is told to drop the binding the daemon abandoned")
12346 .frame;
12347 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12348 assert_eq!(
12349 (goodbye.header.channel, goodbye.header.epoch),
12350 (module_channel, module_epoch)
12351 );
12352 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
12353 }
12354
12355 #[tokio::test]
12360 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
12361 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12362 let client = ConnectionId::new(43);
12363 let (client_ctx, client_rx) = route_ctx(client);
12364
12365 let (open_task, bind) = relay_route_open(
12366 &handler,
12367 client,
12368 &client_ctx.egress,
12369 &mut module_rx,
12370 400,
12371 "aft",
12372 "gone-client-before-mark",
12373 )
12374 .await;
12375 let (module_channel, module_epoch) = route_bind_channel(&bind);
12376 hold.wait().await;
12377
12378 drop(client_rx);
12379 assert!(client_ctx.egress.is_closed());
12380 handler
12381 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12382 .await
12383 .unwrap();
12384
12385 hold.wait().await;
12386 let response = open_task.await.unwrap();
12387 assert_eq!(response.len(), 1);
12388 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12389
12390 let goodbye = module_rx
12391 .try_recv()
12392 .expect("module is told to drop the binding it acked")
12393 .frame;
12394 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12395 assert_eq!(
12396 (goodbye.header.channel, goodbye.header.epoch),
12397 (module_channel, module_epoch)
12398 );
12399 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
12400 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
12401 }
12402
12403 #[test]
12410 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
12411 let handler = ControlHandler::default();
12412 let module_connection = ConnectionId::new(30);
12413 let client_connection = ConnectionId::new(31);
12414
12415 handler
12416 .refuse_to_end_module_connection_for_a_client(
12417 module_connection,
12418 77,
12419 ForwardingError::ConnectionClosing {
12420 connection_id: client_connection,
12421 },
12422 )
12423 .expect("a closing client must never end the module connection");
12424
12425 assert!(matches!(
12426 handler.refuse_to_end_module_connection_for_a_client(
12427 module_connection,
12428 78,
12429 ForwardingError::ConnectionClosing {
12430 connection_id: module_connection,
12431 },
12432 ),
12433 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
12434 connection_id
12435 })) if connection_id == module_connection
12436 ));
12437 assert!(matches!(
12438 handler.refuse_to_end_module_connection_for_a_client(
12439 module_connection,
12440 79,
12441 ForwardingError::Poisoned,
12442 ),
12443 Err(RouterError::Forwarding(ForwardingError::Poisoned))
12444 ));
12445 assert!(matches!(
12446 handler.refuse_to_end_module_connection_for_a_client(
12447 module_connection,
12448 80,
12449 ForwardingError::StaleModuleEndpoint,
12450 ),
12451 Err(RouterError::Forwarding(
12452 ForwardingError::StaleModuleEndpoint
12453 ))
12454 ));
12455 }
12456
12457 #[tokio::test]
12470 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
12471 let registry = Arc::new(Registry::default());
12472 let forwarding = Arc::new(ForwardingTable::default());
12473 let supervisor = SupervisorHandle::new();
12474 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12475 let handler =
12476 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12477 .with_supervisor(supervisor);
12478
12479 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12480 hello_via_sink(
12481 &handler,
12482 &target_ctx,
12483 &mut target_rx,
12484 hello_frame("target", PROTOCOL_VERSION, 1),
12485 )
12486 .await;
12487
12488 let wrong_nonce = handler
12492 .handle_control_frame(
12493 &route_ctx(ConnectionId::new(91)).0,
12494 route_open_frame_with_admission_facts(
12495 20,
12496 "target",
12497 unique_project_root("admission-facts"),
12498 Some(subc_control::ConsumerIdentity {
12499 module_id: "fed".to_string(),
12500 launch_nonce: "not-the-real-nonce".to_string(),
12501 }),
12502 None,
12503 ),
12504 )
12505 .await
12506 .unwrap();
12507 assert_eq!(
12508 parse_error(&wrong_nonce[0])["code"],
12509 "bad_consumer_identity",
12510 "a mismatched launch nonce must be refused, not stamped Reserved"
12511 );
12512
12513 let never_spawned = handler
12517 .handle_control_frame(
12518 &route_ctx(ConnectionId::new(92)).0,
12519 route_open_frame_with_admission_facts(
12520 21,
12521 "target",
12522 unique_project_root("admission-facts"),
12523 Some(subc_control::ConsumerIdentity {
12524 module_id: "never-spawned".to_string(),
12525 launch_nonce: "any-nonce".to_string(),
12526 }),
12527 None,
12528 ),
12529 )
12530 .await
12531 .unwrap();
12532 assert_eq!(
12533 parse_error(&never_spawned[0])["code"],
12534 "bad_consumer_identity",
12535 "an unspawned module_id must be refused rather than accepted for lack of a record"
12536 );
12537 }
12538
12539 #[tokio::test]
12562 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
12563 let registry = Arc::new(Registry::default());
12564 let forwarding = Arc::new(ForwardingTable::default());
12565 let supervisor = SupervisorHandle::new();
12566 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12567 let handler =
12568 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12569 .with_supervisor(supervisor);
12570
12571 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
12572 hello_via_sink(
12573 &handler,
12574 &target_ctx,
12575 &mut target_rx,
12576 hello_frame("target", PROTOCOL_VERSION, 1),
12577 )
12578 .await;
12579
12580 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
12581 let route_handler = handler.clone();
12582 let route_task = tokio::spawn(async move {
12583 route_handler
12584 .handle_control_frame(
12585 &client_ctx,
12586 route_open_frame_with_admission_facts(
12587 30,
12588 "target",
12589 unique_project_root("admission-facts"),
12590 Some(subc_control::ConsumerIdentity {
12591 module_id: "fed".to_string(),
12592 launch_nonce: "fed-nonce".to_string(),
12593 }),
12594 None,
12595 ),
12596 )
12597 .await
12598 .unwrap()
12599 });
12600
12601 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12608 .await
12609 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
12610 .expect("module control channel closed before route.bind");
12611 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12612 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12613 panic!("expected route.bind")
12614 };
12615 assert_eq!(
12616 principal,
12617 Some(Principal::Reserved {
12618 module_id: "fed".to_string()
12619 }),
12620 "a correctly attested consumer must be stamped Reserved for its own id"
12621 );
12622
12623 handler
12624 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12625 .await
12626 .unwrap();
12627 assert!(route_task.await.unwrap().is_empty());
12628 assert!(
12629 matches!(
12630 serde_json::from_slice::<ClientControlResponse>(
12631 &client_rx.recv().await.unwrap().body
12632 )
12633 .unwrap(),
12634 ClientControlResponse::RouteOpen { .. }
12635 ),
12636 "the route must actually open, not merely avoid an error"
12637 );
12638 }
12639
12640 #[tokio::test(start_paused = true)]
12641 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
12642 let registry = Arc::new(Registry::default());
12643 let forwarding = Arc::new(ForwardingTable::default());
12644 let supervisor = SupervisorHandle::new();
12645 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12646 let handler =
12647 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12648 .with_supervisor(supervisor);
12649
12650 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
12651 hello_via_sink(
12652 &handler,
12653 &target_ctx,
12654 &mut target_rx,
12655 hello_frame("target", PROTOCOL_VERSION, 1),
12656 )
12657 .await;
12658
12659 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
12660 let direct_handler = handler.clone();
12661 let direct_open = tokio::spawn(async move {
12662 direct_handler
12663 .handle_control_frame(
12664 &direct_ctx,
12665 route_open_frame(2, "target", unique_project_root("route-census-direct")),
12666 )
12667 .await
12668 .unwrap()
12669 });
12670 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12671 .await
12672 .expect("no direct route.bind within 5s")
12673 .expect("target control channel closed before direct route.bind");
12674 handler
12675 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
12676 .await
12677 .unwrap();
12678 assert!(direct_open.await.unwrap().is_empty());
12679 let _ = direct_rx.recv().await.unwrap();
12680
12681 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
12682 let reserved_handler = handler.clone();
12683 let reserved_open = tokio::spawn(async move {
12684 reserved_handler
12685 .handle_control_frame(
12686 &reserved_ctx,
12687 route_open_frame_with_admission_facts(
12688 3,
12689 "target",
12690 unique_project_root("admission-facts"),
12691 Some(ConsumerIdentity {
12692 module_id: "fed".to_string(),
12693 launch_nonce: "fed-nonce".to_string(),
12694 }),
12695 None,
12696 ),
12697 )
12698 .await
12699 .unwrap()
12700 });
12701 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12702 .await
12703 .expect("no reserved route.bind within 5s")
12704 .expect("target control channel closed before reserved route.bind");
12705 handler
12706 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
12707 .await
12708 .unwrap();
12709 assert!(reserved_open.await.unwrap().is_empty());
12710 let _ = reserved_rx.recv().await.unwrap();
12711
12712 forwarding
12713 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
12714 .unwrap();
12715 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
12716 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
12717 module_id: Some("target".to_string()),
12718 })
12719 .unwrap();
12720 let census_frame =
12721 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
12722 let response = handler
12723 .handle_control_frame(&census_ctx, census_frame)
12724 .await
12725 .unwrap()
12726 .pop()
12727 .unwrap();
12728 let actual: Value = serde_json::from_slice(&response.body).unwrap();
12729 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
12730 assert!(matches!(
12731 decoded,
12732 ClientControlResponse::SupervisorRoutes { .. }
12733 ));
12734 let routes = actual["modules"][0]["routes"].as_array().unwrap();
12735 assert_eq!(routes.len(), 2);
12736 assert!(routes.iter().all(|route| route["draining"] == true));
12737 assert!(
12740 routes.iter().all(|route| route["drain_reason"] == "reload"),
12741 "draining routes must name the drain's reason: {routes:?}"
12742 );
12743 assert!(routes.iter().any(|route| {
12744 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
12745 }));
12746 assert!(routes.iter().any(|route| {
12747 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
12748 }));
12749
12750 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12751 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
12752 if std::env::var_os("UPDATE_GOLDEN").is_some() {
12753 std::fs::write(
12754 &golden_path,
12755 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
12756 )
12757 .unwrap();
12758 }
12759 let expected: Value =
12760 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
12761 assert_eq!(actual, expected);
12762 }
12763
12764 async fn query_live_roots(
12765 handler: &ControlHandler,
12766 module_ctx: &RouteCtx,
12767 ) -> ModuleControlResponseToModule {
12768 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
12769 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
12770 let response = handler
12771 .handle_control_frame(module_ctx, frame)
12772 .await
12773 .unwrap()
12774 .pop()
12775 .unwrap();
12776 serde_json::from_slice(&response.body).unwrap()
12777 }
12778
12779 #[tokio::test(start_paused = true)]
12780 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
12781 let registry = Arc::new(Registry::default());
12782 let forwarding = Arc::new(ForwardingTable::default());
12783 let handler = ControlHandler::with_forwarding(registry, forwarding);
12784 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
12785 hello_via_sink(
12786 &handler,
12787 &target_ctx,
12788 &mut target_rx,
12789 hello_frame("target", PROTOCOL_VERSION, 1),
12790 )
12791 .await;
12792 let root = unique_project_root("live-roots-known");
12793 let path = ProjectRootId::from_path_allowing_missing(root.path())
12794 .unwrap()
12795 .as_path()
12796 .to_path_buf();
12797 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
12798 let open_handler = handler.clone();
12799 let opened = tokio::spawn(async move {
12800 open_handler
12801 .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
12802 .await
12803 .unwrap()
12804 });
12805 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12806 .await
12807 .unwrap()
12808 .unwrap();
12809 handler
12810 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
12811 .await
12812 .unwrap();
12813 assert!(opened.await.unwrap().is_empty());
12814 let _ = client_rx.recv().await.unwrap();
12815
12816 let root = unique_project_root("live-roots-pending");
12817 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
12818 .unwrap()
12819 .as_path()
12820 .to_path_buf();
12821 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
12822 let open_handler = handler.clone();
12823 let pending = tokio::spawn(async move {
12824 open_handler
12825 .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
12826 .await
12827 .unwrap()
12828 });
12829 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12830 .await
12831 .unwrap()
12832 .unwrap();
12833 let actual = query_live_roots(&handler, &target_ctx).await;
12834 let ModuleControlResponseToModule::LiveRoots {
12835 roots,
12836 unknown_root_bindings,
12837 total_bindings,
12838 } = actual
12839 else {
12840 panic!("expected live roots")
12841 };
12842 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
12843 assert_eq!(unknown_root_bindings, 0);
12844 assert_eq!(
12845 roots.len(),
12846 2,
12847 "root-known arm must retain each canonical root"
12848 );
12849 assert_eq!(
12850 total_bindings,
12851 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12852 );
12853 let counts = roots
12854 .iter()
12855 .map(|root| (root.project_root.clone(), root.bound, root.pending))
12856 .collect::<Vec<_>>();
12857 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
12858 expected.sort_by(|a, b| a.0.cmp(&b.0));
12859 assert_eq!(
12860 counts, expected,
12861 "roots must sort by path and count pending separately"
12862 );
12863 handler
12864 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
12865 .await
12866 .unwrap();
12867 assert!(pending.await.unwrap().is_empty());
12868 }
12869
12870 #[tokio::test(start_paused = true)]
12871 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
12872 let forwarding = Arc::new(ForwardingTable::default());
12873 let handler =
12874 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12875 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
12876 hello_via_sink(
12877 &handler,
12878 &target_ctx,
12879 &mut target_rx,
12880 hello_frame("target", PROTOCOL_VERSION, 1),
12881 )
12882 .await;
12883 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
12884 let pending = forwarding
12885 .begin_route_bind_relay_for_test(
12886 client_ctx.connection_id,
12887 client_ctx.egress.clone(),
12888 2,
12889 "target",
12890 )
12891 .unwrap();
12892 forwarding
12893 .complete_pending_relay(
12894 target_ctx.connection_id,
12895 pending.corr,
12896 RouteBindRelayOutcome::Accepted,
12897 )
12898 .unwrap();
12899 let actual = query_live_roots(&handler, &target_ctx).await;
12900 let ModuleControlResponseToModule::LiveRoots {
12901 roots,
12902 unknown_root_bindings,
12903 total_bindings,
12904 } = actual
12905 else {
12906 panic!("expected live roots")
12907 };
12908 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
12909 assert_eq!(
12910 unknown_root_bindings, 1,
12911 "unknown-root arm must not read as no bindings"
12912 );
12913 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
12914 assert_eq!(
12915 total_bindings,
12916 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12917 );
12918 }
12919
12920 #[tokio::test(start_paused = true)]
12928 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
12929 let forwarding = Arc::new(ForwardingTable::default());
12930 let handler =
12931 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12932 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
12933 let replies = handler
12934 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
12935 .await
12936 .unwrap();
12937 let queued_by_hello = module_rx.len();
12938
12939 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
12940 let open_handler = handler.clone();
12941 let open = tokio::spawn(async move {
12942 open_handler
12943 .handle_control_frame(
12944 &client_ctx,
12945 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
12946 )
12947 .await
12948 .unwrap()
12949 });
12950 let mut spins = 0;
12952 while module_rx.len() == queued_by_hello {
12953 spins += 1;
12954 assert!(spins < 10_000, "route.open never queued a route.bind");
12955 tokio::task::yield_now().await;
12956 }
12957
12958 for reply in replies {
12960 module_ctx.egress.send(reply).await.unwrap();
12961 }
12962
12963 let first = module_rx.recv().await.unwrap().frame;
12964 assert_eq!(
12965 first.header.ty,
12966 FrameType::HelloAck,
12967 "the first frame a registering module reads must be its HELLO_ACK"
12968 );
12969 assert_eq!(first.header.corr, 7);
12970 let second = module_rx.recv().await.unwrap().frame;
12971 assert_eq!(second.header.ty, FrameType::Request);
12972 assert!(
12973 matches!(
12974 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
12975 ModuleControlRequest::RouteBind { .. }
12976 ),
12977 "the route.bind follows the ack"
12978 );
12979 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
12980
12981 handler
12982 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
12983 .await
12984 .unwrap();
12985 assert!(open.await.unwrap().is_empty());
12986 let _ = client_rx.recv().await.unwrap();
12987 }
12988
12989 #[tokio::test(start_paused = true)]
12990 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
12991 let handler = ControlHandler::with_forwarding(
12992 Arc::new(Registry::default()),
12993 Arc::new(ForwardingTable::default()),
12994 );
12995 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
12996 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
12997 hello_via_sink(
12998 &handler,
12999 &first_ctx,
13000 &mut first_rx,
13001 hello_frame("first", PROTOCOL_VERSION, 1),
13002 )
13003 .await;
13004 hello_via_sink(
13005 &handler,
13006 &second_ctx,
13007 &mut second_rx,
13008 hello_frame("second", PROTOCOL_VERSION, 2),
13009 )
13010 .await;
13011 let root = unique_project_root("second-only");
13012 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
13013 let cloned = handler.clone();
13014 let open = tokio::spawn(async move {
13015 cloned
13016 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
13017 .await
13018 .unwrap()
13019 });
13020 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
13021 .await
13022 .unwrap()
13023 .unwrap();
13024 let first = query_live_roots(&handler, &first_ctx).await;
13025 let second = query_live_roots(&handler, &second_ctx).await;
13026 assert!(
13027 matches!(
13028 first,
13029 ModuleControlResponseToModule::LiveRoots {
13030 total_bindings: 0,
13031 ..
13032 }
13033 ),
13034 "cross-module scope must not expose another module's roots"
13035 );
13036 assert!(
13037 matches!(
13038 second,
13039 ModuleControlResponseToModule::LiveRoots {
13040 total_bindings: 1,
13041 ..
13042 }
13043 ),
13044 "second module must see its pending route"
13045 );
13046 handler
13047 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
13048 .await
13049 .unwrap();
13050 assert!(open.await.unwrap().is_empty());
13051 }
13052
13053 #[tokio::test(start_paused = true)]
13054 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
13055 let handler = ControlHandler::with_forwarding(
13056 Arc::new(Registry::default()),
13057 Arc::new(ForwardingTable::default()),
13058 );
13059 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
13060 hello_via_sink(
13061 &handler,
13062 &target_ctx,
13063 &mut target_rx,
13064 hello_frame("target", PROTOCOL_VERSION, 1),
13065 )
13066 .await;
13067 let actual = query_live_roots(&handler, &target_ctx).await;
13068 let ModuleControlResponseToModule::LiveRoots {
13069 roots,
13070 unknown_root_bindings,
13071 total_bindings,
13072 } = actual
13073 else {
13074 panic!("expected live roots")
13075 };
13076 assert!(roots.is_empty());
13077 assert_eq!(unknown_root_bindings, 0);
13078 assert_eq!(total_bindings, 0);
13079 assert_eq!(
13080 total_bindings,
13081 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
13082 );
13083 }
13084
13085 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
13092 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
13093 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
13094 let text = std::fs::read_to_string(&path)
13095 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
13096 let vectors: Vec<(String, Value)> = text
13097 .lines()
13098 .filter(|line| !line.trim().is_empty())
13099 .map(|line| {
13100 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
13101 let id = entry["corpus_id"]
13102 .as_str()
13103 .expect("every vector carries a corpus_id")
13104 .to_string();
13105 (id, entry["package"].clone())
13106 })
13107 .collect();
13108 assert_eq!(
13112 vectors.len(),
13113 3,
13114 "vendored fed corpus changed size; re-sync from subc-federation"
13115 );
13116
13117 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
13132 let richest = vectors
13133 .iter()
13134 .filter_map(|(_, package)| package.as_object())
13135 .map(|object| {
13136 object
13137 .keys()
13138 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
13139 .count()
13140 })
13141 .max()
13142 .unwrap_or(0);
13143 assert!(
13144 richest >= 2,
13145 "vendored corpus no longer carries a package with unmodelled fields, \
13146 so the relay test can no longer distinguish a verbatim relay from a lossy one"
13147 );
13148
13149 vectors
13150 }
13151
13152 #[tokio::test]
13160 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
13161 for (corpus_id, package) in fed_admission_facts_vectors() {
13162 let registry = Arc::new(Registry::default());
13163 let forwarding = Arc::new(ForwardingTable::default());
13164 let supervisor = SupervisorHandle::new();
13165 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
13166 let handler =
13167 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13168 .with_supervisor(supervisor)
13169 .with_admission_facts_config(
13170 Some("fed".to_string()),
13171 Some(vec!["target".to_string()]),
13172 );
13173
13174 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
13175 hello_via_sink(
13176 &handler,
13177 &target_ctx,
13178 &mut target_rx,
13179 hello_frame("target", PROTOCOL_VERSION, 1),
13180 )
13181 .await;
13182
13183 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
13184 let route_handler = handler.clone();
13185 let expected = package.clone();
13186 let route_task = tokio::spawn(async move {
13187 route_handler
13188 .handle_control_frame(
13189 &client_ctx,
13190 route_open_frame_with_admission_facts(
13191 20,
13192 "target",
13193 unique_project_root("admission-facts"),
13194 Some(subc_control::ConsumerIdentity {
13195 module_id: "fed".to_string(),
13196 launch_nonce: "fed-nonce".to_string(),
13197 }),
13198 Some(package),
13199 ),
13200 )
13201 .await
13202 .unwrap()
13203 });
13204
13205 let bind_frame = target_rx.recv().await.unwrap();
13206 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13207 let ModuleControlRequest::RouteBind {
13208 admission_facts, ..
13209 } = bind
13210 else {
13211 panic!("{corpus_id}: expected route.bind")
13212 };
13213 assert_eq!(
13214 admission_facts,
13215 Some(expected),
13216 "{corpus_id}: relay must not add, drop or reshape any field"
13217 );
13218
13219 handler
13220 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13221 .await
13222 .unwrap();
13223 route_task.await.unwrap();
13224 }
13225 }
13226
13227 #[tokio::test]
13228 async fn route_bind_never_relays_the_openers_build_class() {
13229 let registry = Arc::new(Registry::default());
13230 let forwarding = Arc::new(ForwardingTable::default());
13231 let handler =
13232 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13233 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
13234 hello_via_sink(
13235 &handler,
13236 &target_ctx,
13237 &mut target_rx,
13238 hello_frame("target", PROTOCOL_VERSION, 1),
13239 )
13240 .await;
13241
13242 let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
13243 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
13244 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
13245 busy_refusals: None,
13246 target: RouteTarget::ToolProvider {
13247 module_id: "target".to_string(),
13248 },
13249 identity,
13250 consumer_identity: None,
13251 consumer_capabilities: None,
13252 role_versions: None,
13253 admission_facts: None,
13254 scope: None,
13255 })
13256 .unwrap();
13257 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
13258
13259 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
13260 let route_handler = handler.clone();
13261 let route_task = tokio::spawn(async move {
13262 route_handler
13263 .handle_control_frame(&client_ctx, open)
13264 .await
13265 .unwrap()
13266 });
13267 let bind_frame = target_rx.recv().await.unwrap();
13268 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13269 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
13270 panic!("expected route.bind")
13271 };
13272 assert_eq!(
13273 identity.build_class, None,
13274 "route.bind carried the build class"
13275 );
13276 assert!(
13277 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
13278 "the route.bind bytes still name build_class"
13279 );
13280 handler
13281 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13282 .await
13283 .unwrap();
13284 route_task.await.unwrap();
13285 }
13286
13287 #[tokio::test]
13288 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
13289 let registry = Arc::new(Registry::default());
13290 let forwarding = Arc::new(ForwardingTable::default());
13291 let supervisor = SupervisorHandle::new();
13292 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
13293 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
13294 let handler =
13295 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13296 .with_supervisor(supervisor)
13297 .with_admission_facts_config(
13298 Some("fed".to_string()),
13299 Some(vec!["target".to_string()]),
13300 );
13301
13302 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
13303 hello_via_sink(
13304 &handler,
13305 &target_ctx,
13306 &mut target_rx,
13307 hello_frame("target", PROTOCOL_VERSION, 1),
13308 )
13309 .await;
13310 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
13311 hello_via_sink(
13312 &handler,
13313 &other_ctx,
13314 &mut other_rx,
13315 hello_frame("other", PROTOCOL_VERSION, 2),
13316 )
13317 .await;
13318
13319 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
13320 let expected_facts = facts.clone();
13321 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
13322 let route_handler = handler.clone();
13323 let route_task = tokio::spawn(async move {
13324 route_handler
13325 .handle_control_frame(
13326 &client_ctx,
13327 route_open_frame_with_admission_facts(
13328 10,
13329 "target",
13330 unique_project_root("admission-facts"),
13331 Some(subc_control::ConsumerIdentity {
13332 module_id: "fed".to_string(),
13333 launch_nonce: "fed-nonce".to_string(),
13334 }),
13335 Some(facts.clone()),
13336 ),
13337 )
13338 .await
13339 .unwrap()
13340 });
13341 let bind_frame = target_rx.recv().await.unwrap();
13342 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13343 let ModuleControlRequest::RouteBind {
13344 admission_facts, ..
13345 } = bind
13346 else {
13347 panic!("expected route.bind")
13348 };
13349 assert_eq!(admission_facts, Some(expected_facts));
13350 handler
13351 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13352 .await
13353 .unwrap();
13354 assert!(route_task.await.unwrap().is_empty());
13355 assert!(matches!(
13356 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
13357 .unwrap(),
13358 ClientControlResponse::RouteOpen { .. }
13359 ));
13360
13361 let direct = handler
13362 .handle_control_frame(
13363 &route_ctx(ConnectionId::new(73)).0,
13364 route_open_frame_with_admission_facts(
13365 11,
13366 "target",
13367 unique_project_root("admission-facts"),
13368 None,
13369 Some(json!({"x": 1})),
13370 ),
13371 )
13372 .await
13373 .unwrap();
13374 assert_eq!(
13375 parse_error(&direct[0])["code"],
13376 "admission_facts_not_permitted"
13377 );
13378
13379 let different_reserved = handler
13380 .handle_control_frame(
13381 &route_ctx(ConnectionId::new(77)).0,
13382 route_open_frame_with_admission_facts(
13383 15,
13384 "target",
13385 unique_project_root("admission-facts"),
13386 Some(subc_control::ConsumerIdentity {
13387 module_id: "other".to_string(),
13388 launch_nonce: "other-nonce".to_string(),
13389 }),
13390 Some(json!({"x": 1})),
13391 ),
13392 )
13393 .await
13394 .unwrap();
13395 assert_eq!(
13396 parse_error(&different_reserved[0])["code"],
13397 "admission_facts_not_permitted"
13398 );
13399
13400 let other_target = handler
13401 .handle_control_frame(
13402 &route_ctx(ConnectionId::new(74)).0,
13403 route_open_frame_with_admission_facts(
13404 12,
13405 "other",
13406 unique_project_root("admission-facts"),
13407 Some(subc_control::ConsumerIdentity {
13408 module_id: "fed".to_string(),
13409 launch_nonce: "fed-nonce".to_string(),
13410 }),
13411 Some(json!({"x": 1})),
13412 ),
13413 )
13414 .await
13415 .unwrap();
13416 assert_eq!(
13417 parse_error(&other_target[0])["code"],
13418 "admission_facts_target_not_allowed"
13419 );
13420
13421 let nonexistent = handler
13422 .handle_control_frame(
13423 &route_ctx(ConnectionId::new(75)).0,
13424 route_open_frame_with_admission_facts(
13425 13,
13426 "missing",
13427 unique_project_root("admission-facts"),
13428 None,
13429 Some(json!({"x": 1})),
13430 ),
13431 )
13432 .await
13433 .unwrap();
13434 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
13435
13436 let described = handler
13437 .handle_control_frame(
13438 &route_ctx(ConnectionId::new(76)).0,
13439 Frame::build(
13440 FrameType::Request,
13441 control_flags(),
13442 0,
13443 0,
13444 14,
13445 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13446 )
13447 .unwrap(),
13448 )
13449 .await
13450 .unwrap();
13451 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13452 serde_json::from_slice(&described[0].body).unwrap()
13453 else {
13454 panic!("expected server.describe response")
13455 };
13456 assert!(capabilities
13457 .iter()
13458 .any(|cap| cap == "admission_facts_relay_v1"));
13459 }
13460
13461 #[tokio::test]
13462 async fn admission_facts_without_configured_carrier_are_rejected() {
13463 let registry = Arc::new(Registry::default());
13464 let forwarding = Arc::new(ForwardingTable::default());
13465 let handler = ControlHandler::with_forwarding(registry, forwarding);
13466 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
13467 hello_via_sink(
13468 &handler,
13469 &target_ctx,
13470 &mut target_rx,
13471 hello_frame("target", PROTOCOL_VERSION, 1),
13472 )
13473 .await;
13474
13475 let responses = handler
13476 .handle_control_frame(
13477 &route_ctx(ConnectionId::new(79)).0,
13478 route_open_frame_with_admission_facts(
13479 16,
13480 "target",
13481 unique_project_root("admission-facts"),
13482 None,
13483 Some(json!({"x": 1})),
13484 ),
13485 )
13486 .await
13487 .unwrap();
13488 assert_eq!(
13489 parse_error(&responses[0])["code"],
13490 "admission_facts_not_permitted"
13491 );
13492 }
13493
13494 #[tokio::test]
13495 async fn route_open_relays_consumer_capabilities_verbatim() {
13496 let registry = Arc::new(Registry::default());
13497 let forwarding = Arc::new(ForwardingTable::default());
13498 let handler =
13499 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13500 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
13501 hello_via_sink(
13502 &handler,
13503 &module_ctx,
13504 &mut module_rx,
13505 hello_frame("aft", PROTOCOL_VERSION, 7),
13506 )
13507 .await;
13508
13509 let expected = vec!["elicitation".to_string(), "roots".to_string()];
13510 let expected_for_request = expected.clone();
13511 let project_root = unique_project_root("consumer-capabilities-present");
13512 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
13513 let route_handler = handler.clone();
13514 let route_task = tokio::spawn(async move {
13515 route_handler
13516 .handle_control_frame(
13517 &client_ctx,
13518 route_open_frame_with_consumer_capabilities(
13519 401,
13520 "aft",
13521 project_root,
13522 Some(expected_for_request),
13523 ),
13524 )
13525 .await
13526 .unwrap()
13527 });
13528 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13529 .await
13530 .unwrap()
13531 .unwrap();
13532 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13533 let ModuleControlRequest::RouteBind {
13534 consumer_capabilities,
13535 ..
13536 } = bind
13537 else {
13538 panic!("expected route.bind request, got {bind:?}");
13539 };
13540 assert_eq!(consumer_capabilities, Some(expected.clone()));
13541
13542 handler
13543 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13544 .await
13545 .unwrap();
13546 let route_response = route_task.await.unwrap();
13547 assert!(route_response.is_empty());
13548 let published = client_rx.recv().await.unwrap();
13549 assert!(matches!(
13550 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13551 ClientControlResponse::RouteOpen { .. }
13552 ));
13553 }
13554
13555 #[tokio::test]
13556 async fn route_open_without_consumer_capabilities_relays_none() {
13557 let registry = Arc::new(Registry::default());
13558 let forwarding = Arc::new(ForwardingTable::default());
13559 let handler =
13560 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13561 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
13562 hello_via_sink(
13563 &handler,
13564 &module_ctx,
13565 &mut module_rx,
13566 hello_frame("aft", PROTOCOL_VERSION, 7),
13567 )
13568 .await;
13569
13570 let project_root = unique_project_root("consumer-capabilities-absent");
13571 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
13572 let route_handler = handler.clone();
13573 let route_task = tokio::spawn(async move {
13574 route_handler
13575 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
13576 .await
13577 .unwrap()
13578 });
13579 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13580 .await
13581 .unwrap()
13582 .unwrap();
13583 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13584 let ModuleControlRequest::RouteBind {
13585 consumer_capabilities,
13586 ..
13587 } = bind
13588 else {
13589 panic!("expected route.bind request, got {bind:?}");
13590 };
13591 assert_eq!(consumer_capabilities, None);
13592
13593 handler
13594 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13595 .await
13596 .unwrap();
13597 let route_response = route_task.await.unwrap();
13598 assert!(route_response.is_empty());
13599 let published = client_rx.recv().await.unwrap();
13600 assert!(matches!(
13601 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13602 ClientControlResponse::RouteOpen { .. }
13603 ));
13604 }
13605
13606 async fn bind_role_versions_for(
13610 sent: Option<BTreeMap<String, String>>,
13611 connection: u64,
13612 ) -> Option<BTreeMap<String, String>> {
13613 let registry = Arc::new(Registry::default());
13614 let forwarding = Arc::new(ForwardingTable::default());
13615 let handler =
13616 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13617 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
13618 hello_via_sink(
13619 &handler,
13620 &module_ctx,
13621 &mut module_rx,
13622 hello_frame("aft", PROTOCOL_VERSION, 7),
13623 )
13624 .await;
13625 let project_root = unique_project_root("role-versions");
13626 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
13627 let route_handler = handler.clone();
13628 let route_task = tokio::spawn(async move {
13629 route_handler
13630 .handle_control_frame(
13631 &client_ctx,
13632 route_open_frame_with_role_versions(403, "aft", project_root, sent),
13633 )
13634 .await
13635 .unwrap()
13636 });
13637 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13638 .await
13639 .expect("a well-formed route.open reaches the module as a bind")
13640 .unwrap();
13641 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13642 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
13643 panic!("expected route.bind request, got {bind:?}");
13644 };
13645 handler
13646 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13647 .await
13648 .unwrap();
13649 assert!(route_task.await.unwrap().is_empty());
13650 let published = client_rx.recv().await.unwrap();
13651 assert!(matches!(
13652 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13653 ClientControlResponse::RouteOpen { .. }
13654 ));
13655 role_versions
13656 }
13657
13658 #[tokio::test]
13659 async fn route_open_relays_role_versions_verbatim() {
13660 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
13661 assert_eq!(
13662 bind_role_versions_for(Some(sent.clone()), 141).await,
13663 Some(sent)
13664 );
13665 }
13666
13667 #[tokio::test]
13670 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
13671 assert_eq!(bind_role_versions_for(None, 143).await, None);
13672 assert_eq!(
13673 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
13674 None
13675 );
13676 }
13677
13678 #[tokio::test]
13679 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
13680 let registry = Arc::new(Registry::default());
13681 let forwarding = Arc::new(ForwardingTable::default());
13682 let handler =
13683 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13684 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
13685 hello_via_sink(
13686 &handler,
13687 &module_ctx,
13688 &mut module_rx,
13689 hello_frame("aft", PROTOCOL_VERSION, 7),
13690 )
13691 .await;
13692
13693 let nine: BTreeMap<String, String> = (0..9)
13694 .map(|index| (format!("role-{index}"), "v1".to_string()))
13695 .collect();
13696 for (label, malformed) in [
13697 (
13698 "invalid role name",
13699 role_versions(&[("Tool_Provider", "v1")]),
13700 ),
13701 ("invalid version", role_versions(&[("tool-provider", "v0")])),
13702 ("nine entries", nine),
13703 ] {
13704 let responses = tokio::time::timeout(
13707 Duration::from_secs(1),
13708 handler.handle_control_frame(
13709 &route_ctx(ConnectionId::new(148)).0,
13710 route_open_frame_with_role_versions(
13711 404,
13712 "aft",
13713 unique_project_root("role-versions-malformed"),
13714 Some(malformed),
13715 ),
13716 ),
13717 )
13718 .await
13719 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
13720 .unwrap();
13721 assert_eq!(responses.len(), 1, "{label}");
13722 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
13723 let error = parse_error(&responses[0]);
13724 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
13725 assert_eq!(
13726 error["detail"]["field"], "role_versions",
13727 "{label}: {error}"
13728 );
13729 assert!(
13730 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
13731 "{label}: a malformed declaration is terminal"
13732 );
13733 assert!(
13734 module_rx.try_recv().is_err(),
13735 "{label}: the module must never see a bind"
13736 );
13737 }
13738 }
13739
13740 #[tokio::test]
13744 async fn route_role_versions_capability_is_advertised() {
13745 let handler = ControlHandler::new(Arc::new(Registry::default()));
13746 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
13747 let ack = hello_via_sink(
13748 &handler,
13749 &ctx,
13750 &mut rx,
13751 hello_frame("m", PROTOCOL_VERSION, 1),
13752 )
13753 .await;
13754 let ack = parse_ack(&ack);
13755 assert!(
13756 ack.subc_capabilities
13757 .iter()
13758 .any(|c| c == "route-role-versions/v1"),
13759 "{:?}",
13760 ack.subc_capabilities
13761 );
13762
13763 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13764 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13765 let reply = handler
13766 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
13767 .await
13768 .unwrap()
13769 .pop()
13770 .unwrap();
13771 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13772 serde_json::from_slice(&reply.body).unwrap()
13773 else {
13774 panic!("not a server.describe reply");
13775 };
13776 assert!(
13777 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
13778 "{capabilities:?}"
13779 );
13780 }
13781
13782 #[tokio::test]
13786 async fn catalog_events_capability_is_advertised() {
13787 let handler = ControlHandler::new(Arc::new(Registry::default()));
13788 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13789 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13790 let reply = handler
13791 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
13792 .await
13793 .unwrap()
13794 .pop()
13795 .unwrap();
13796 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13797 serde_json::from_slice(&reply.body).unwrap()
13798 else {
13799 panic!("not a server.describe reply");
13800 };
13801 assert!(
13802 capabilities
13803 .iter()
13804 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
13805 "{capabilities:?}"
13806 );
13807 }
13808
13809 #[tokio::test]
13810 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
13811 let handler = ControlHandler::new(Arc::new(Registry::default()));
13812 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
13813 let ack = hello_via_sink(
13814 &handler,
13815 &module,
13816 &mut module_rx,
13817 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
13818 )
13819 .await;
13820 let ack = parse_ack(&ack);
13821 assert!(
13822 ack.subc_capabilities
13823 .iter()
13824 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13825 "{:?}",
13826 ack.subc_capabilities
13827 );
13828 assert!(ack
13829 .subc_capabilities
13830 .iter()
13831 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13832
13833 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
13834 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13835 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13836 let reply = handler
13837 .handle_control_frame(&client, frame)
13838 .await
13839 .unwrap()
13840 .pop()
13841 .unwrap();
13842 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13843 serde_json::from_slice(&reply.body).unwrap()
13844 else {
13845 panic!("not a server.describe reply");
13846 };
13847 assert!(
13848 capabilities
13849 .iter()
13850 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13851 "{capabilities:?}"
13852 );
13853 assert!(capabilities
13854 .iter()
13855 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13856 }
13857
13858 #[tokio::test]
13859 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
13860 let handler = ControlHandler::new(Arc::new(Registry::default()));
13861 let (module, mut rx) = route_ctx(ConnectionId::new(156));
13862 let ack = hello_via_sink(
13863 &handler,
13864 &module,
13865 &mut rx,
13866 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
13867 )
13868 .await;
13869 assert!(parse_ack(&ack)
13870 .subc_capabilities
13871 .iter()
13872 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13873 assert!(parse_ack(&ack)
13874 .subc_capabilities
13875 .iter()
13876 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13877 let (client, _rx) = route_ctx(ConnectionId::new(157));
13878 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13879 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13880 let reply = handler
13881 .handle_control_frame(&client, frame)
13882 .await
13883 .unwrap()
13884 .pop()
13885 .unwrap();
13886 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13887 serde_json::from_slice(&reply.body).unwrap()
13888 else {
13889 panic!("server.describe reply")
13890 };
13891 assert!(capabilities
13892 .iter()
13893 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13894 assert!(capabilities
13895 .iter()
13896 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13897 }
13898
13899 #[tokio::test]
13900 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
13901 let handler = ControlHandler::new(Arc::new(Registry::default()));
13902 let (module, mut rx) = route_ctx(ConnectionId::new(158));
13903 let ack = hello_via_sink(
13904 &handler,
13905 &module,
13906 &mut rx,
13907 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
13908 )
13909 .await;
13910 assert!(parse_ack(&ack)
13911 .subc_capabilities
13912 .iter()
13913 .any(|cap| cap == "scope-workdir/v1"));
13914 let (client, _rx) = route_ctx(ConnectionId::new(159));
13915 let frame = Frame::build(
13916 FrameType::Request,
13917 control_flags(),
13918 0,
13919 0,
13920 5,
13921 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13922 )
13923 .unwrap();
13924 let reply = handler
13925 .handle_control_frame(&client, frame)
13926 .await
13927 .unwrap()
13928 .remove(0);
13929 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13930 serde_json::from_slice(&reply.body).unwrap()
13931 else {
13932 panic!("server.describe reply")
13933 };
13934 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
13935 }
13936
13937 #[tokio::test]
13938 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
13939 let handler = ControlHandler::new(Arc::new(Registry::default()));
13940 let (module, mut rx) = route_ctx(ConnectionId::new(160));
13941 let ack = hello_via_sink(
13942 &handler,
13943 &module,
13944 &mut rx,
13945 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
13946 )
13947 .await;
13948 assert!(parse_ack(&ack)
13949 .subc_capabilities
13950 .iter()
13951 .any(|cap| cap == "worker-runs/v1"));
13952 let (client, _rx) = route_ctx(ConnectionId::new(161));
13953 let frame = Frame::build(
13954 FrameType::Request,
13955 control_flags(),
13956 0,
13957 0,
13958 5,
13959 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13960 )
13961 .unwrap();
13962 let reply = handler
13963 .handle_control_frame(&client, frame)
13964 .await
13965 .unwrap()
13966 .remove(0);
13967 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13968 serde_json::from_slice(&reply.body).unwrap()
13969 else {
13970 panic!("server.describe reply")
13971 };
13972 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
13973 }
13974
13975 #[tokio::test]
13976 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
13977 let handler = ControlHandler::new(Arc::new(Registry::default()));
13978 let (module, mut rx) = route_ctx(ConnectionId::new(154));
13979 let ack = hello_via_sink(
13980 &handler,
13981 &module,
13982 &mut rx,
13983 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
13984 )
13985 .await;
13986 assert!(parse_ack(&ack)
13987 .subc_capabilities
13988 .iter()
13989 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13990 let (client, _rx) = route_ctx(ConnectionId::new(155));
13991 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13992 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13993 let reply = handler
13994 .handle_control_frame(&client, frame)
13995 .await
13996 .unwrap()
13997 .pop()
13998 .unwrap();
13999 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14000 serde_json::from_slice(&reply.body).unwrap()
14001 else {
14002 panic!("not a server.describe reply")
14003 };
14004 assert!(capabilities
14005 .iter()
14006 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
14007 }
14008
14009 #[tokio::test]
14010 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
14011 let registry = Arc::new(Registry::default());
14012 let forwarding = Arc::new(ForwardingTable::default());
14013 let handler =
14014 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14015 .with_health_probe_timeout(Duration::from_secs(5));
14016 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
14017 hello_via_sink(
14018 &handler,
14019 &module_ctx,
14020 &mut module_rx,
14021 non_routable_hello_frame_with_control_ops(
14022 "mcp",
14023 300,
14024 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14025 ),
14026 )
14027 .await;
14028 assert!(registry
14029 .get_module("mcp")
14030 .unwrap()
14031 .unwrap()
14032 .manifest
14033 .provides
14034 .is_empty());
14035
14036 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
14037 let route_response = handler
14038 .handle_control_frame(
14039 &route_client_ctx,
14040 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
14041 )
14042 .await
14043 .unwrap();
14044 assert_eq!(route_response[0].header.ty, FrameType::Error);
14045 assert_eq!(
14046 parse_error(&route_response[0])["code"],
14047 "target_unavailable"
14048 );
14049 assert!(parse_error(&route_response[0])["message"]
14050 .as_str()
14051 .unwrap()
14052 .contains("does not provide the requested target"));
14053 assert!(module_rx.try_recv().is_err());
14054
14055 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
14056 let health_handler = handler.clone();
14057 let health_task = tokio::spawn(async move {
14058 health_handler
14059 .handle_control_frame(
14060 &health_client_ctx,
14061 supervisor_health_probe_frame(302, "mcp"),
14062 )
14063 .await
14064 .unwrap()
14065 });
14066 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
14067 .await
14068 .unwrap()
14069 .unwrap();
14070 assert_eq!(
14071 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
14072 ModuleControlRequest::HealthCheck {}
14073 );
14074 handler
14075 .handle_control_frame(
14076 &module_ctx,
14077 health_response(health_frame.header.corr, HealthStatus::Ok),
14078 )
14079 .await
14080 .unwrap();
14081 let health_response = health_task.await.unwrap();
14082 assert_eq!(health_response[0].header.ty, FrameType::Response);
14083 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
14084 ClientControlResponse::SupervisorHealthProbe {
14085 module_id, status, ..
14086 } => {
14087 assert_eq!(module_id, "mcp");
14088 assert_eq!(status, HealthStatus::Ok);
14089 }
14090 other => panic!("unexpected health response: {other:?}"),
14091 }
14092
14093 forwarding
14098 .cleanup_connection(module_ctx.connection_id)
14099 .unwrap();
14100 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
14101 let cleanup_response = tokio::time::timeout(
14102 Duration::from_millis(200),
14103 handler.handle_control_frame(
14104 &cleanup_probe_ctx,
14105 supervisor_health_probe_frame(303, "mcp"),
14106 ),
14107 )
14108 .await
14109 .expect("probe should fail immediately when the control lane is gone")
14110 .unwrap();
14111 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
14112 assert_eq!(
14113 parse_error(&cleanup_response[0])["code"],
14114 "target_unavailable"
14115 );
14116 assert!(parse_error(&cleanup_response[0])["message"]
14117 .as_str()
14118 .unwrap()
14119 .contains("no module connection"));
14120
14121 handler
14122 .cleanup_connection(module_ctx.connection_id)
14123 .unwrap();
14124 }
14125
14126 #[tokio::test]
14127 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
14128 let registry = Arc::new(Registry::default());
14129 let supervisor_handle = SupervisorHandle::new();
14130 let supervisor =
14131 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
14132 .with_handle(supervisor_handle.clone())
14133 .with_connection_file_path(
14134 std::env::temp_dir()
14135 .join(format!("subc-route-open-warming-{}", std::process::id())),
14136 );
14137 let module = supervisor
14138 .supervise_configured(
14139 ModuleSpec {
14140 carry_containment: false,
14141 module_id: "warming".to_string(),
14142 program: fake_aft_stub_path(),
14143 args: Vec::new(),
14144 env: Vec::new(),
14145 reserved: false,
14146 reserved_prefixes: Vec::new(),
14147 protocol: ModuleProtocol::Subc,
14148 overlap: Default::default(),
14149 },
14150 true,
14151 )
14152 .unwrap();
14153 assert_eq!(module.state().unwrap(), ModuleState::Running);
14154
14155 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14156 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
14157 let response = handler
14158 .handle_control_frame(
14159 &ctx,
14160 route_open_frame(304, "warming", unique_project_root("warming")),
14161 )
14162 .await
14163 .unwrap();
14164 module.stop().await.unwrap();
14165
14166 assert_eq!(response[0].header.ty, FrameType::Error);
14167 let error = parse_error(&response[0]);
14168 assert_eq!(error["code"], "module_warming");
14169 assert!(error["message"]
14170 .as_str()
14171 .unwrap()
14172 .contains("state=running, enabled=true, live=false"));
14173 }
14174
14175 #[test]
14176 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
14177 let handler = ControlHandler::new(Arc::new(Registry::default()));
14178 let capture = EventCapture::default();
14179 let _subscriber =
14180 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14181 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
14182 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
14183 let pending = (0..limit).collect::<Vec<_>>();
14184 let response = handler
14185 .route_open_capacity_refusal(
14186 &ctx,
14187 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
14188 "busy",
14189 pending.len(),
14190 limit,
14191 )
14192 .unwrap();
14193 assert_eq!(parse_error(&response)["code"], "target_unavailable");
14194 let event = capture
14195 .events()
14196 .into_iter()
14197 .find(|event| {
14198 event.target == "control"
14199 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
14200 })
14201 .expect("connection admission refusal event");
14202 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
14203 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
14204 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14207 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14208 }
14209
14210 #[test]
14211 fn route_open_target_cap_logs_admission_reason_and_capacity() {
14212 let handler = ControlHandler::new(Arc::new(Registry::default()));
14213 let capture = EventCapture::default();
14214 let _subscriber =
14215 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14216 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
14217 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
14218 let guards = (0..limit)
14219 .map(|_| {
14220 handler
14221 .route_bind_concurrency
14222 .try_admit("busy", limit)
14223 .unwrap()
14224 })
14225 .collect::<Vec<_>>();
14226 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
14227 Err(in_flight) => in_flight,
14228 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
14229 };
14230 let response = handler
14231 .route_open_target_capacity_refusal(
14232 &ctx,
14233 &RouteOpenCaller::unknown(),
14234 &route_open_frame(397, "busy", unique_project_root("target-cap")),
14235 "busy",
14236 in_flight,
14237 )
14238 .unwrap();
14239 assert_eq!(parse_error(&response)["code"], "target_unavailable");
14240 let event = capture
14241 .events()
14242 .into_iter()
14243 .find(|event| {
14244 event.target == "control"
14245 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
14246 })
14247 .expect("target admission refusal event");
14248 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
14249 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
14250 drop(guards);
14251 }
14252
14253 #[cfg(unix)]
14254 #[tokio::test(start_paused = true)]
14255 async fn supervisor_list_shows_a_failed_modules_next_retry() {
14256 let registry = Arc::new(Registry::default());
14257 let handle = SupervisorHandle::new();
14258 let supervisor =
14259 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
14260 .with_handle(handle.clone());
14261 let module = supervisor
14262 .spawn(crate::ModuleSpec {
14263 carry_containment: false,
14264 module_id: "retry-list".into(),
14265 program: PathBuf::from("/bin/sh"),
14266 args: vec!["-c".into(), "exit 17".into()],
14267 env: vec![],
14268 reserved: false,
14269 reserved_prefixes: vec![],
14270 protocol: ModuleProtocol::None,
14271 overlap: Default::default(),
14272 })
14273 .unwrap();
14274 let deadline = std::time::Instant::now() + Duration::from_secs(10);
14275 while module.state().unwrap() != crate::ModuleState::Failed {
14276 assert!(
14277 std::time::Instant::now() < deadline,
14278 "waiting for budget failure"
14279 );
14280 tokio::task::yield_now().await;
14281 }
14282 let handler = ControlHandler::new(registry).with_supervisor(handle);
14283 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14284 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14285 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14286 assert_eq!(body["modules"][0]["state"], "failed");
14287 let retry = body["modules"][0]["next_retry_at_ms"]
14288 .as_u64()
14289 .expect("scheduled retry on the wire");
14290 let now = std::time::SystemTime::now()
14291 .duration_since(std::time::UNIX_EPOCH)
14292 .unwrap()
14293 .as_millis() as u64;
14294 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
14295 module.set_enabled(false).await.unwrap();
14296 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14297 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14298 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14299 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
14300 }
14301
14302 #[tokio::test]
14303 async fn slow_step_before_pid_read_does_not_spend_its_budget() {
14304 let _guard = crate::subtree_resources::fault_read_guard().await;
14305 let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
14306 let supervisor =
14307 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14308 let module = supervisor
14309 .spawn(crate::ModuleSpec {
14310 carry_containment: false,
14311 module_id: "pid-read-budget".into(),
14312 program: std::env::current_exe().unwrap(),
14313 args: vec![
14314 "--exact".into(),
14315 "control::tests::pid_resource_read_fixture".into(),
14316 ],
14317 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14318 .into_iter()
14319 .map(|key| (key.into(), dir.display().to_string()))
14320 .chain(std::iter::once((
14321 "SUBC_SUBTREE_PID_FIXTURE".into(),
14322 "1".into(),
14323 )))
14324 .collect(),
14325 reserved: false,
14326 reserved_prefixes: Vec::new(),
14327 protocol: ModuleProtocol::None,
14328 overlap: Default::default(),
14329 })
14330 .unwrap();
14331 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
14332 while !matches!(
14333 module.child_resource_usage(),
14334 subc_control::ChildResourceUsage::Measured(_)
14335 ) {
14336 assert!(
14337 tokio::time::Instant::now() < ready_deadline,
14338 "fixture process was not measurable"
14339 );
14340 tokio::time::sleep(Duration::from_millis(10)).await;
14341 }
14342 tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
14343 let reading = crate::subtree_resources::read_process(module.clone()).await;
14344 module.stop().await.unwrap();
14345 assert!(
14346 matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
14347 "{reading:?}"
14348 );
14349 }
14350
14351 #[test]
14352 fn pid_resource_read_fixture() {
14353 if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
14354 loop {
14355 std::thread::park_timeout(Duration::from_secs(600));
14356 }
14357 }
14358 }
14359
14360 #[tokio::test]
14361 async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
14362 let registry = Arc::new(Registry::default());
14363 let handle = SupervisorHandle::new();
14364 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14365 .with_handle(handle.clone());
14366 supervisor
14367 .supervise_configured(
14368 crate::ModuleSpec {
14369 module_id: "disabled-resource-reading".into(),
14370 program: std::env::current_exe().unwrap(),
14371 args: Vec::new(),
14372 env: Vec::new(),
14373 reserved: false,
14374 reserved_prefixes: Vec::new(),
14375 protocol: ModuleProtocol::None,
14376 overlap: Default::default(),
14377 carry_containment: false,
14378 },
14379 false,
14380 )
14381 .unwrap();
14382 let handler = ControlHandler::new(registry).with_supervisor(handle);
14383 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14384 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14385 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14386 assert!(body["modules"][0].get("tree").is_none());
14387 assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
14388 }
14389
14390 #[tokio::test]
14391 async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
14392 let _guard = crate::subtree_resources::fault_read_guard().await;
14393 let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
14394 let registry = Arc::new(Registry::default());
14395 let handle = SupervisorHandle::new();
14396 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14397 .with_handle(handle.clone());
14398 let module = supervisor
14399 .spawn(crate::ModuleSpec {
14400 carry_containment: false,
14401 module_id: "list-process-priority".into(),
14402 program: std::env::current_exe().unwrap(),
14403 args: vec![
14404 "--exact".into(),
14405 "control::tests::pid_resource_read_fixture".into(),
14406 ],
14407 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14408 .into_iter()
14409 .map(|key| (key.into(), dir.display().to_string()))
14410 .chain(std::iter::once((
14411 "SUBC_SUBTREE_PID_FIXTURE".into(),
14412 "1".into(),
14413 )))
14414 .collect(),
14415 reserved: false,
14416 reserved_prefixes: Vec::new(),
14417 protocol: ModuleProtocol::None,
14418 overlap: Default::default(),
14419 })
14420 .unwrap();
14421 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
14422 loop {
14423 let image = module.running_image_agreement().await;
14424 if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
14425 break;
14426 }
14427 assert!(
14428 tokio::time::Instant::now() < ready_deadline,
14429 "the fixture process never became observable: {image:?}"
14430 );
14431 tokio::time::sleep(Duration::from_millis(10)).await;
14432 }
14433 let (target, release, _) = crate::subtree_resources::held();
14434 module.set_subtree_target_for_test(target);
14435 let handler = ControlHandler::new(registry).with_supervisor(handle);
14436 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14437 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14438 drop(release);
14439 module.stop().await.unwrap();
14440 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14441 assert_eq!(
14442 body["modules"][0]["resources"]["status"], "measured",
14443 "the optional tree timeout must not discard the process reading: {body}"
14444 );
14445 assert!(body["modules"][0].get("tree").is_none());
14446 assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
14447 assert_eq!(
14448 body["modules"][0]["pending_reload"]["image"]["status"], "match",
14449 "the tree timeout must not hide a completed image observation"
14450 );
14451 }
14452
14453 #[cfg(any(target_os = "linux", target_os = "macos"))]
14454 #[tokio::test]
14455 async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
14456 let mut handler = ControlHandler::new(Arc::new(Registry::default()))
14457 .with_daemon_provenance(
14458 std::process::id(),
14459 0,
14460 Some(std::env::current_exe().unwrap()),
14461 None,
14462 None,
14463 );
14464 let (probe, release, began) = ExecutableIdentityProbe::held();
14465 handler.daemon_provenance.probe = probe;
14466 let observe = |handler: ControlHandler, corr| {
14467 tokio::spawn(async move {
14468 let (ctx, _) = route_ctx(ConnectionId::new(corr));
14469 let frame =
14470 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
14471 .unwrap();
14472 handler
14473 .handle_supervisor_provenance(&ctx, frame, None)
14474 .await
14475 .unwrap()
14476 })
14477 };
14478 let explicit = observe(handler.clone(), 114);
14479 began.await.unwrap();
14480 let concurrent = observe(handler.clone(), 115);
14481 tokio::time::sleep(Duration::from_millis(300)).await;
14484 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14485 let list = tokio::time::timeout(
14486 Duration::from_secs(1),
14487 handler.handle_supervisor_list(frame),
14488 )
14489 .await;
14490 let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
14491 drop(release);
14492 let replies = explicit.await.unwrap();
14493 let concurrent_replies = concurrent.await.unwrap();
14494 assert!(
14495 explicit_was_pending,
14496 "explicit provenance must await the full observation"
14497 );
14498 let list = list
14499 .expect("a concurrent explicit probe must not stall list")
14500 .unwrap();
14501 let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
14502 assert_eq!(list_body["op"], "supervisor.list");
14503 for replies in [replies, concurrent_replies] {
14504 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14505 assert_eq!(
14506 body["daemon"]["daemon_observed"]["running_image"]["status"],
14507 "match"
14508 );
14509 }
14510 }
14511
14512 #[tokio::test]
14513 async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
14514 let _guard = crate::subtree_resources::fault_read_guard().await;
14515 let registry = Arc::new(Registry::default());
14516 let handle = SupervisorHandle::new();
14517 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14518 .with_handle(handle.clone());
14519 let module = supervisor
14520 .supervise_configured(
14521 crate::ModuleSpec {
14522 carry_containment: false,
14523 module_id: "memory-read".into(),
14524 program: PathBuf::from("disabled-module"),
14525 args: Vec::new(),
14526 env: Vec::new(),
14527 reserved: false,
14528 reserved_prefixes: Vec::new(),
14529 protocol: ModuleProtocol::Subc,
14530 overlap: Default::default(),
14531 },
14532 true,
14533 )
14534 .unwrap();
14535 let (target, release, began) = crate::subtree_resources::held();
14536 let stalled_target = target.clone();
14539 let stalled = tokio::task::spawn_blocking(move || {
14540 stalled_target.tree(&subc_os::ProcessTable::default())
14541 });
14542 began.await.unwrap();
14543 module.set_subtree_target_for_test(target);
14544 let handler = ControlHandler::new(registry).with_supervisor(handle);
14545 let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
14546 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14547 let replies = handler
14548 .handle_client_control_request(
14549 &ctx,
14550 frame,
14551 ClientControlRequest::SupervisorList {},
14552 &mut None,
14553 )
14554 .await
14555 .unwrap();
14556 assert!(
14557 replies.is_empty(),
14558 "list reply must be detached from the connection reader"
14559 );
14560 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14561 let replies = handler
14562 .handle_client_control_request(
14563 &ctx,
14564 frame,
14565 ClientControlRequest::SupervisorSpawnSnapshot {},
14566 &mut None,
14567 )
14568 .await
14569 .unwrap();
14570 assert_eq!(
14571 replies.len(),
14572 1,
14573 "another request must complete while the subtree read stalls"
14574 );
14575 let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
14576 .await
14577 .unwrap()
14578 .unwrap();
14579 release.send(()).unwrap();
14580 drop(release);
14581 stalled.await.unwrap();
14582 let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
14583 assert!(
14584 body["modules"][0].get("tree").is_none(),
14585 "an unknown tree reading must be omitted after the bounded read expires"
14586 );
14587 handler
14591 .resource_tracker
14592 .read(handler.supervisor.list(), false, "released_tree_fixture")
14593 .await;
14594 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14595 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14596 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14597 assert_eq!(
14598 body["modules"][0]["tree"]["source"], "process_tree",
14599 "the next request must export the completed refresh"
14600 );
14601 assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
14602 }
14603
14604 #[tokio::test]
14605 async fn supervisor_list_expired_budget_keeps_committed_accounting_rows() {
14606 use subc_control::{
14607 AgentCommandResources, ChildMemoryKind, ChildResourceReading,
14608 ChildResourceUnavailableReason, ChildResourceUsage, CpuLastHour, DaemonResources,
14609 TreeReading, TreeSource,
14610 };
14611 let registry = Arc::new(Registry::default());
14612 let handle = SupervisorHandle::new();
14613 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14614 .with_handle(handle.clone());
14615 supervisor
14616 .supervise_configured(
14617 crate::ModuleSpec {
14618 carry_containment: false,
14619 module_id: "budget-worker".into(),
14620 program: PathBuf::from("disabled-module"),
14621 args: Vec::new(),
14622 env: Vec::new(),
14623 reserved: false,
14624 reserved_prefixes: Vec::new(),
14625 protocol: ModuleProtocol::Subc,
14626 overlap: Default::default(),
14627 },
14628 true,
14629 )
14630 .unwrap();
14631 let handler = ControlHandler::new(registry).with_supervisor(handle);
14632 let now = u64::try_from(
14633 std::time::SystemTime::now()
14634 .duration_since(std::time::UNIX_EPOCH)
14635 .unwrap()
14636 .as_millis(),
14637 )
14638 .unwrap();
14639 let measured = now - 1_000;
14640 let hour = CpuLastHour::new(100, 60_000);
14641 let daemon = DaemonResources::new(7, 1, "test")
14642 .with_measured_at_ms(Some(measured))
14643 .with_resources(Some(ChildResourceUsage::Measured(
14644 ChildResourceReading::new(20, ChildMemoryKind::ResidentSet, 10, 0),
14645 )))
14646 .with_cpu_last_hour(Some(hour.clone()));
14647 let agents = AgentCommandResources::new("budget-worker", 50, 30, 1)
14648 .with_measured_at_ms(Some(measured))
14649 .with_cpu_last_hour(Some(hour.clone()));
14650 let tree = TreeReading::new(
14651 150,
14652 50,
14653 ChildMemoryKind::ResidentSet,
14654 1,
14655 TreeSource::ProcessTree,
14656 )
14657 .with_measured_at_ms(Some(measured));
14658 handler.resource_tracker.seed_committed_for_test(
14659 &handler.supervisor.list(),
14660 crate::resource_totals::Readings {
14661 modules: BTreeMap::from([("budget-worker".into(), (Some(tree), Some(hour)))]),
14662 daemon: Some(daemon),
14663 agent_commands: Some(vec![agents]),
14664 ..Default::default()
14665 },
14666 now,
14667 );
14668 list_diagnostics::scope(async {
14669 tokio::time::sleep(Duration::from_millis(110)).await;
14670 assert!(list_diagnostics::deadline().unwrap() <= tokio::time::Instant::now());
14671 let request =
14672 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14673 let replies = handler.handle_supervisor_list_inner(request).await.unwrap();
14674 let ClientControlResponse::SupervisorList {
14675 modules,
14676 daemon,
14677 agent_commands,
14678 ..
14679 } = serde_json::from_slice(&replies[0].body).unwrap()
14680 else {
14681 panic!("expected a list response")
14682 };
14683 assert!(matches!(
14684 modules[0].resources,
14685 Some(ChildResourceUsage::Unavailable {
14686 reason: ChildResourceUnavailableReason::Unreadable
14687 })
14688 ));
14689 assert_eq!(
14690 (
14691 modules[0].tree.as_ref().unwrap().cpu_ms,
14692 modules[0].tree.as_ref().unwrap().measured_at_ms
14693 ),
14694 (150, Some(measured))
14695 );
14696 let daemon = daemon.unwrap();
14697 let agents = agent_commands.unwrap();
14698 assert_eq!(daemon.measured_at_ms, Some(measured));
14699 assert_eq!(
14700 (agents.len(), agents[0].cpu_ms, agents[0].measured_at_ms),
14701 (1, 50, Some(measured))
14702 );
14703 for hour in [
14704 modules[0].cpu_last_hour.as_ref(),
14705 daemon.cpu_last_hour.as_ref(),
14706 agents[0].cpu_last_hour.as_ref(),
14707 ] {
14708 let hour = hour.unwrap();
14709 assert_eq!((hour.cpu_ms, hour.covered_ms), (100, 60_000));
14710 }
14711 })
14712 .await;
14713 }
14714
14715 #[tokio::test]
14716 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
14717 let registry = Arc::new(Registry::default());
14718 let handle = SupervisorHandle::new();
14719 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14720 .with_handle(handle.clone());
14721 supervisor
14722 .supervise_configured(
14723 crate::ModuleSpec {
14724 carry_containment: false,
14725 module_id: "drift".into(),
14726 program: PathBuf::from("disabled-module"),
14727 args: Vec::new(),
14728 env: Vec::new(),
14729 reserved: false,
14730 reserved_prefixes: Vec::new(),
14731 protocol: ModuleProtocol::Subc,
14732 overlap: Default::default(),
14733 },
14734 false,
14735 )
14736 .unwrap();
14737 let handler = ControlHandler::new(registry).with_supervisor(handle);
14738 for configured in [None, Some(false), Some(true)] {
14739 handler.capability_evaluator.configure(
14740 configured.map(|enabled| ("drift".to_string(), enabled)),
14741 BTreeMap::new(),
14742 );
14743 let frame =
14744 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14745 let replies = handler
14746 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
14747 .await
14748 .unwrap();
14749 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14750 assert!(body.get("configured_enabled").is_none());
14751 assert_eq!(body["op"], "supervisor.ack");
14752 let frame =
14753 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14754 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14755 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14756 assert_eq!(
14757 body["modules"][0].get("configured_enabled"),
14758 configured.map(Value::Bool).as_ref()
14759 );
14760 assert!(body["modules"][0].get("tree").is_none());
14761 assert!(body["modules"][0].get("subtree").is_none());
14762 }
14763 }
14764
14765 #[tokio::test(flavor = "current_thread")]
14766 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
14767 let handler = ControlHandler::new(Arc::new(Registry::default()));
14768 let capture = EventCapture::default();
14769 let _subscriber =
14770 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14771 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
14772 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14773
14774 let _ = handler
14775 .handle_client_control_request(
14776 &ctx,
14777 frame,
14778 ClientControlRequest::SupervisorSetEnabled {
14779 module_id: "broca".to_string(),
14780 enabled: false,
14781 },
14782 &mut None,
14783 )
14784 .await
14785 .unwrap();
14786
14787 let events = capture
14788 .events()
14789 .into_iter()
14790 .filter(|event| {
14791 event.target == "control"
14792 && event.fields.get("message").map(String::as_str)
14793 == Some("supervisor request received")
14794 })
14795 .collect::<Vec<_>>();
14796 assert_eq!(events.len(), 1, "one supervisor request log line");
14797 let event = &events[0];
14798 assert_eq!(event.level, tracing::Level::INFO);
14799 assert_eq!(
14800 event.fields.get("op"),
14801 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
14802 );
14803 assert_eq!(
14804 event.fields.get("module_id"),
14805 Some(&"\"broca\"".to_string())
14806 );
14807 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
14808 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
14809 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
14810 }
14811
14812 #[tokio::test(flavor = "current_thread")]
14813 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
14814 let registry = Arc::new(Registry::default());
14815 let module_connection = ConnectionId::new(303);
14816 registry
14817 .register_with_control_ops(
14818 manifest("aft", PROTOCOL_VERSION),
14819 PROTOCOL_VERSION,
14820 module_connection,
14821 module_baseline_control_ops(),
14822 )
14823 .unwrap();
14824 let handler = ControlHandler::new(registry);
14825 let capture = EventCapture::default();
14826 let _subscriber =
14827 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14828 let (ctx, _rx) = route_ctx(module_connection);
14829 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14830
14831 let _ = handler
14832 .handle_client_control_request(
14833 &ctx,
14834 frame,
14835 ClientControlRequest::SupervisorSetEnabled {
14836 module_id: "broca".to_string(),
14837 enabled: false,
14838 },
14839 &mut None,
14840 )
14841 .await
14842 .unwrap();
14843
14844 let callers = capture
14845 .events()
14846 .into_iter()
14847 .filter(|event| {
14848 event.fields.get("message").map(String::as_str)
14849 == Some("supervisor request received")
14850 })
14851 .map(|event| event.fields.get("caller").cloned())
14852 .collect::<Vec<_>>();
14853 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
14854 }
14855
14856 #[tokio::test(flavor = "current_thread")]
14857 async fn supervisor_list_does_not_log_request_received() {
14858 let handler = ControlHandler::new(Arc::new(Registry::default()));
14859 let capture = EventCapture::default();
14860 let _subscriber =
14861 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14862 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
14863 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14864
14865 handler
14866 .handle_client_control_request(
14867 &ctx,
14868 frame,
14869 ClientControlRequest::SupervisorList {},
14870 &mut None,
14871 )
14872 .await
14873 .unwrap();
14874
14875 assert!(capture.events().into_iter().all(|event| {
14876 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14877 }));
14878 }
14879
14880 #[tokio::test(flavor = "current_thread")]
14881 async fn supervisor_rescan_preview_does_not_log_request_received() {
14882 let handler = ControlHandler::new(Arc::new(Registry::default()));
14883 let capture = EventCapture::default();
14884 let _subscriber =
14885 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14886 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
14887 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14888
14889 handler
14890 .handle_client_control_request(
14891 &ctx,
14892 frame,
14893 ClientControlRequest::SupervisorRescan { preview: true },
14894 &mut None,
14895 )
14896 .await
14897 .unwrap();
14898
14899 assert!(capture.events().into_iter().all(|event| {
14900 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14901 }));
14902 }
14903
14904 struct ApproveOperator;
14905 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
14906 fn prompt(
14907 &self,
14908 _: &str,
14909 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
14910 ) -> crate::operator_confirm::ProviderResult {
14911 crate::operator_confirm::ProviderResult::Approved
14912 }
14913 }
14914
14915 #[tokio::test]
14916 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
14917 assert_eq!(
14918 ControlHandler::default().privileged_operator_prompt,
14919 PrivilegedOperatorPrompt::Log
14920 );
14921 for (initial, changed, running) in [
14922 ("log", "enforce", PrivilegedOperatorPrompt::Log),
14923 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
14924 ] {
14925 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
14926 let path = dir.join("subc.jsonc");
14927 std::fs::write(
14928 &path,
14929 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
14930 )
14931 .unwrap();
14932 let config = crate::daemon_config::load(&path).unwrap().unwrap();
14933 let registry = Arc::new(Registry::default());
14934 let handle = SupervisorHandle::new();
14935 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
14936 .with_handle(handle.clone());
14937 let handler = ControlHandler::new(registry)
14938 .with_supervisor(handle)
14939 .with_privileged_operator_prompt(config.privileged_operator_prompt)
14940 .with_supervisor_rescan(supervisor, &path, config.port);
14941 handler.forwarding.operator_confirms().configure(
14942 Arc::new(ApproveOperator),
14943 crate::operator_confirm::OperatorLimits::default(),
14944 );
14945 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
14946
14947 for (disk_mode, needs_restart) in [
14948 (Some(initial), false),
14949 (Some(changed), true),
14950 (Some(changed), true),
14951 (None, running == PrivilegedOperatorPrompt::Enforce),
14952 (Some(initial), false),
14953 ] {
14954 let mut doc = json!({"version":1});
14955 if let Some(mode) = disk_mode {
14956 doc["privileged_operator_prompt"] = json!(mode);
14957 }
14958 std::fs::write(&path, doc.to_string()).unwrap();
14959 for preview in [true, false] {
14960 let request = ClientControlRequest::SupervisorRescan { preview };
14961 let frame = Frame::build(
14962 FrameType::Request,
14963 control_flags(),
14964 0,
14965 0,
14966 1,
14967 serde_json::to_vec(&request).unwrap(),
14968 )
14969 .unwrap();
14970 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14971 assert_eq!(frames.len(), 1);
14972 let ClientControlResponse::SupervisorRescan { result } =
14973 serde_json::from_slice(&frames[0].body).unwrap()
14974 else {
14975 panic!("expected a rescan response");
14976 };
14977 let expected = if needs_restart {
14978 vec!["privileged_operator_prompt"]
14979 } else {
14980 Vec::new()
14981 };
14982 assert_eq!(result.restart_required, expected);
14983 assert_eq!(handler.privileged_operator_prompt, running);
14984 assert_eq!(
14985 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
14986 running
14987 );
14988 }
14989 }
14990 }
14991 }
14992
14993 #[tokio::test]
14994 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
14995 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
14996 let path = dir.join("subc.jsonc");
14997 let registry = Arc::new(Registry::default());
14998 let handle = SupervisorHandle::new();
14999 let supervisor =
15000 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
15001 let handler = ControlHandler::new(registry)
15002 .with_supervisor(handle)
15003 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
15004 .with_supervisor_rescan(supervisor, &path, None);
15005 handler.forwarding.operator_confirms().configure(
15006 Arc::new(ApproveOperator),
15007 crate::operator_confirm::OperatorLimits::default(),
15008 );
15009 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
15010 for preview in [true, false] {
15011 std::fs::write(
15012 &path,
15013 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
15014 )
15015 .unwrap();
15016 let request = ClientControlRequest::SupervisorRescan { preview };
15017 let frame = Frame::build(
15018 FrameType::Request,
15019 control_flags(),
15020 0,
15021 0,
15022 1,
15023 serde_json::to_vec(&request).unwrap(),
15024 )
15025 .unwrap();
15026 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
15027 assert_eq!(frames.len(), 1);
15028 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
15029 assert_eq!(
15030 handler.privileged_operator_prompt,
15031 PrivilegedOperatorPrompt::Enforce
15032 );
15033 }
15034 }
15035
15036 #[tokio::test]
15041 async fn route_open_refusal_names_the_check_that_refused() {
15042 let handler = ControlHandler::new(Arc::new(Registry::default()));
15043 let capture = EventCapture::default();
15044 let _subscriber =
15045 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15046 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15047 let response = handler
15048 .handle_control_frame(
15049 &ctx,
15050 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
15051 )
15052 .await
15053 .unwrap();
15054
15055 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15056 let event = capture
15057 .events()
15058 .into_iter()
15059 .find(|event| {
15060 event.target == "control"
15061 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
15062 })
15063 .expect("route.open refusal event");
15064 assert_eq!(
15065 event.fields.get("reason"),
15066 Some(&"\"not_registered\"".to_string())
15067 );
15068 }
15069
15070 #[tokio::test]
15071 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
15072 let registry = Arc::new(Registry::default());
15073 let supervisor_handle = SupervisorHandle::new();
15074 let supervisor =
15075 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
15076 .with_handle(supervisor_handle.clone())
15077 .with_connection_file_path(std::env::temp_dir().join(format!(
15078 "subc-route-open-refusal-info-{}",
15079 std::process::id()
15080 )));
15081 let module = supervisor
15082 .supervise_configured(
15083 ModuleSpec {
15084 carry_containment: false,
15085 module_id: "warming".to_string(),
15086 program: fake_aft_stub_path(),
15087 args: Vec::new(),
15088 env: Vec::new(),
15089 reserved: false,
15090 reserved_prefixes: Vec::new(),
15091 protocol: ModuleProtocol::Subc,
15092 overlap: Default::default(),
15093 },
15094 true,
15095 )
15096 .unwrap();
15097 assert_eq!(module.state().unwrap(), ModuleState::Running);
15098
15099 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15100 assert!(handler
15101 .counters()
15102 .snapshot()
15103 .get("route_open_refused_by_code")
15104 .is_none());
15105 let capture = EventCapture::default();
15106 let _subscriber =
15107 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15108 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
15109 let response = handler
15110 .handle_control_frame(
15111 &ctx,
15112 route_open_frame(394, "warming", unique_project_root("refusal-info")),
15113 )
15114 .await
15115 .unwrap();
15116 module.stop().await.unwrap();
15117
15118 assert_eq!(parse_error(&response[0])["code"], "module_warming");
15119 let event = capture
15120 .events()
15121 .into_iter()
15122 .find(|event| {
15123 event.target == "control"
15124 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
15125 })
15126 .expect("route.open refusal event");
15127 assert_eq!(
15128 event.fields.get("module_id"),
15129 Some(&"\"warming\"".to_string())
15130 );
15131 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
15132 assert_eq!(
15133 event.fields.get("reason"),
15134 Some(&"\"supervised_not_registered\"".to_string())
15135 );
15136 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
15137 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
15138 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
15139 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
15140 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
15141 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
15142 assert_eq!(
15143 handler.counters().snapshot()["route_open_refused_by_code"],
15144 json!({ "module_warming": 1 })
15145 );
15146 }
15147
15148 const OUTAGE_START: &str = "route.open refusing module: not serving";
15149 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
15150
15151 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
15152 capture
15153 .events()
15154 .into_iter()
15155 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
15156 .collect()
15157 }
15158
15159 fn supervise_stub(
15160 registry: &Arc<Registry>,
15161 module_id: &str,
15162 enabled: bool,
15163 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
15164 let supervisor_handle = SupervisorHandle::new();
15165 let supervisor =
15166 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
15167 .with_handle(supervisor_handle.clone())
15168 .with_connection_file_path(std::env::temp_dir().join(format!(
15169 "subc-route-outage-{module_id}-{}",
15170 std::process::id()
15171 )));
15172 let module = supervisor
15173 .supervise_configured(
15174 ModuleSpec {
15175 carry_containment: false,
15176 module_id: module_id.to_string(),
15177 program: fake_aft_stub_path(),
15178 args: Vec::new(),
15179 env: Vec::new(),
15180 reserved: false,
15181 reserved_prefixes: Vec::new(),
15182 protocol: ModuleProtocol::Subc,
15183 overlap: Default::default(),
15184 },
15185 enabled,
15186 )
15187 .unwrap();
15188 (supervisor_handle, module)
15189 }
15190
15191 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
15192 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
15193 module_id: module_id.to_string(),
15194 drain_timeout_ms: Some(50),
15195 })
15196 .unwrap();
15197 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
15198 }
15199
15200 #[test]
15204 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
15205 let registry = Arc::new(Registry::default());
15206 let forwarding = Arc::new(ForwardingTable::default());
15207 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15208 let second = ControlHandler::with_forwarding(registry, forwarding);
15209 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
15210 }
15211
15212 #[tokio::test(flavor = "current_thread")]
15216 async fn route_open_unknown_module_refusals_add_no_outage_state() {
15217 let handler = ControlHandler::new(Arc::new(Registry::default()));
15218 let capture = EventCapture::default();
15219 let _subscriber =
15220 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15221 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
15222 for corr in 0..8 {
15223 let response = handler
15224 .handle_control_frame(
15225 &ctx,
15226 route_open_frame(
15227 380 + corr,
15228 &format!("nobody-{corr}"),
15229 unique_project_root("outage-unknown"),
15230 ),
15231 )
15232 .await
15233 .unwrap();
15234 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15235 }
15236
15237 assert_eq!(handler.route_outages.tracked_module_count(), 0);
15238 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
15239 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
15240 }
15241
15242 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15247 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
15248 let registry = Arc::new(Registry::default());
15249 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
15250 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15251 let capture = EventCapture::default();
15252 let _subscriber =
15253 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15254 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
15255 handler.route_outages.record_accepted("outage-restart");
15258
15259 let response = handler
15260 .handle_control_frame(
15261 &ctx,
15262 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
15263 )
15264 .await
15265 .unwrap();
15266 assert_eq!(response[0].header.ty, FrameType::Error);
15267 let starts = outage_lines(&capture, OUTAGE_START);
15268 assert_eq!(starts.len(), 1, "{starts:?}");
15269 assert_eq!(starts[0].level, tracing::Level::WARN);
15270 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
15271 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
15272 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
15273 handler.route_outages.record_accepted("outage-restart");
15274 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
15275
15276 let restart = handler
15277 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
15278 .await
15279 .unwrap();
15280 assert_eq!(
15281 restart[0].header.ty,
15282 FrameType::Response,
15283 "{:?}",
15284 parse_error(&restart[0])
15285 );
15286 handler
15287 .handle_control_frame(
15288 &ctx,
15289 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
15290 )
15291 .await
15292 .unwrap();
15293 module.stop().await.unwrap();
15294
15295 let starts = outage_lines(&capture, OUTAGE_START);
15296 assert_eq!(starts.len(), 2, "{starts:?}");
15297 assert_eq!(starts[1].level, tracing::Level::INFO);
15298 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
15299 }
15300
15301 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15305 async fn failed_operator_restart_leaves_no_operator_mark() {
15306 let registry = Arc::new(Registry::default());
15307 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
15308 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15309 let capture = EventCapture::default();
15310 let _subscriber =
15311 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15312 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
15313 handler.route_outages.record_accepted("outage-disabled");
15314
15315 let restart = handler
15316 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
15317 .await
15318 .unwrap();
15319 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
15320 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
15321
15322 handler
15323 .handle_control_frame(
15324 &ctx,
15325 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
15326 )
15327 .await
15328 .unwrap();
15329 let starts = outage_lines(&capture, OUTAGE_START);
15330 assert_eq!(starts.len(), 1, "{starts:?}");
15331 assert_eq!(starts[0].level, tracing::Level::WARN);
15332 }
15333
15334 #[tokio::test(flavor = "current_thread")]
15335 async fn route_open_unknown_module_escapes_target_module_id() {
15336 let handler = ControlHandler::new(Arc::new(Registry::default()));
15337 let capture = EventCapture::default();
15338 let _subscriber =
15339 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15340 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
15341 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15342 let response = handler
15343 .handle_control_frame(
15344 &ctx,
15345 route_open_frame(
15346 395,
15347 hostile_module_id,
15348 unique_project_root("hostile-target-module-id"),
15349 ),
15350 )
15351 .await
15352 .unwrap();
15353
15354 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15355 let event = capture
15356 .events()
15357 .into_iter()
15358 .find(|event| {
15359 event.target == "control"
15360 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
15361 })
15362 .expect("route.open unknown-module refusal event");
15363 let logged = event.fields.get("module_id").expect("module_id field");
15364 assert!(!logged.bytes().any(|byte| byte < 0x20));
15365 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
15366 }
15367
15368 #[tokio::test(flavor = "current_thread")]
15371 async fn route_open_refusal_carries_daemon_origin() {
15372 let handler = ControlHandler::new(Arc::new(Registry::default()));
15373 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15374 let response = handler
15375 .handle_control_frame(
15376 &ctx,
15377 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
15378 )
15379 .await
15380 .unwrap();
15381
15382 assert_eq!(response.len(), 1);
15383 assert_eq!(response[0].header.ty, FrameType::Error);
15384 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15385 assert!(
15386 response[0].header.flags.is_daemon_origin(),
15387 "route.open refusal must carry daemon origin: {:?}",
15388 response[0].header.flags
15389 );
15390 }
15391
15392 #[tokio::test(flavor = "current_thread")]
15396 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
15397 let registry = Arc::new(Registry::default());
15398 let forwarding = Arc::new(ForwardingTable::default());
15399 let handler =
15400 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15401 let module_connection = ConnectionId::new(95);
15402 let (module_ctx, mut module_rx) = route_ctx(module_connection);
15403 hello_via_sink(
15404 &handler,
15405 &module_ctx,
15406 &mut module_rx,
15407 hello_frame("aft", PROTOCOL_VERSION, 395),
15408 )
15409 .await;
15410 forwarding
15411 .begin_module_drain("aft", RouteCloseReason::Restart)
15412 .unwrap()
15413 .expect("the module endpoint drains");
15414
15415 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
15416 let response = handler
15417 .handle_control_frame(
15418 &client_ctx,
15419 route_open_frame(396, "aft", unique_project_root("drain-origin")),
15420 )
15421 .await
15422 .unwrap();
15423
15424 assert_eq!(response.len(), 1);
15425 assert_eq!(response[0].header.ty, FrameType::Error);
15426 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15427 assert!(
15428 response[0].header.flags.is_daemon_origin(),
15429 "drain module_reloading must carry daemon origin: {:?}",
15430 response[0].header.flags
15431 );
15432 }
15433
15434 #[tokio::test(flavor = "current_thread")]
15438 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
15439 let registry = Arc::new(Registry::default());
15440 let forwarding = Arc::new(ForwardingTable::default());
15441 let handler =
15442 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15443 let module_connection = ConnectionId::new(95);
15444 let (module_ctx, mut module_rx) = route_ctx(module_connection);
15445 hello_via_sink(
15446 &handler,
15447 &module_ctx,
15448 &mut module_rx,
15449 hello_frame("aft", PROTOCOL_VERSION, 395),
15450 )
15451 .await;
15452
15453 let client_connection = ConnectionId::new(96);
15454 let (client_ctx, _client_rx) = route_ctx(client_connection);
15455 let (route_task, bind) = relay_route_open(
15456 &handler,
15457 client_connection,
15458 &client_ctx.egress,
15459 &mut module_rx,
15460 396,
15461 "aft",
15462 "spoofed-reloading",
15463 )
15464 .await;
15465 let rejection = Frame::build(
15468 FrameType::Error,
15469 crate::router::daemon_error_flags(),
15470 0,
15471 0,
15472 bind.header.corr,
15473 serde_json::to_vec(&ErrorBody::new(
15474 "module_reloading",
15475 "module refused route.bind",
15476 ))
15477 .unwrap(),
15478 )
15479 .unwrap();
15480 handler
15481 .handle_control_frame(&module_ctx, rejection)
15482 .await
15483 .unwrap();
15484
15485 let response = route_task.await.unwrap();
15486 assert_eq!(response.len(), 1);
15487 assert_eq!(response[0].header.ty, FrameType::Error);
15488 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15489 assert!(
15490 !response[0].header.flags.is_daemon_origin(),
15491 "a module's own rejection must not carry daemon origin: {:?}",
15492 response[0].header.flags
15493 );
15494 }
15495
15496 #[tokio::test(flavor = "current_thread")]
15497 async fn route_open_module_rejection_uses_daemon_counter_key() {
15498 let registry = Arc::new(Registry::default());
15499 let forwarding = Arc::new(ForwardingTable::default());
15500 let handler =
15501 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15502 let module_connection = ConnectionId::new(95);
15503 let (module_ctx, mut module_rx) = route_ctx(module_connection);
15504 hello_via_sink(
15505 &handler,
15506 &module_ctx,
15507 &mut module_rx,
15508 hello_frame("aft", PROTOCOL_VERSION, 395),
15509 )
15510 .await;
15511
15512 let client_connection = ConnectionId::new(96);
15513 let (client_ctx, _client_rx) = route_ctx(client_connection);
15514 let capture = EventCapture::default();
15515 let _subscriber =
15516 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15517 let (route_task, bind) = relay_route_open(
15518 &handler,
15519 client_connection,
15520 &client_ctx.egress,
15521 &mut module_rx,
15522 396,
15523 "aft",
15524 "hostile-module-code",
15525 )
15526 .await;
15527 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
15528 let rejection = Frame::build(
15529 FrameType::Error,
15530 control_flags(),
15531 0,
15532 0,
15533 bind.header.corr,
15534 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
15535 )
15536 .unwrap();
15537 handler
15538 .handle_control_frame(&module_ctx, rejection)
15539 .await
15540 .unwrap();
15541
15542 let response = route_task.await.unwrap();
15543 assert_eq!(parse_error(&response[0])["code"], hostile_code);
15544 let counters = handler.counters().snapshot();
15545 assert_eq!(
15546 counters["route_open_refused_by_code"],
15547 json!({ "module_rejected": 1 })
15548 );
15549 assert!(counters["route_open_refused_by_code"]
15550 .get(hostile_code)
15551 .is_none());
15552
15553 let event = capture
15554 .events()
15555 .into_iter()
15556 .find(|event| {
15557 event.target == "control"
15558 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
15559 })
15560 .expect("route.open module-rejection refusal event");
15561 let logged = event.fields.get("module_code").expect("module_code field");
15562 assert!(!logged.bytes().any(|byte| byte < 0x20));
15563 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
15564 }
15565
15566 #[tokio::test]
15567 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
15568 let registry = Arc::new(Registry::default());
15569 let supervisor_handle = SupervisorHandle::new();
15570 let missing_program = std::env::temp_dir().join(format!(
15571 "subc-route-open-missing-program-{}",
15572 std::process::id()
15573 ));
15574 let supervisor =
15575 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
15576 .with_handle(supervisor_handle.clone());
15577 let module = supervisor
15578 .supervise_configured(
15579 ModuleSpec {
15580 carry_containment: false,
15581 module_id: "failed".to_string(),
15582 program: missing_program,
15583 args: Vec::new(),
15584 env: Vec::new(),
15585 reserved: false,
15586 reserved_prefixes: Vec::new(),
15587 protocol: ModuleProtocol::Subc,
15588 overlap: Default::default(),
15589 },
15590 true,
15591 )
15592 .unwrap();
15593 assert_eq!(module.state().unwrap(), ModuleState::Failed);
15594
15595 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15596 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
15597 let response = handler
15598 .handle_control_frame(
15599 &ctx,
15600 route_open_frame(305, "failed", unique_project_root("failed")),
15601 )
15602 .await
15603 .unwrap();
15604
15605 assert_eq!(response[0].header.ty, FrameType::Error);
15606 let error = parse_error(&response[0]);
15607 assert_eq!(error["code"], "target_unavailable");
15608 assert!(error["message"]
15609 .as_str()
15610 .unwrap()
15611 .contains("state=failed, enabled=true, live=false"));
15612 }
15613
15614 #[tokio::test]
15615 async fn route_open_role_mismatch_remains_target_unavailable() {
15616 let registry = Arc::new(Registry::default());
15617 let handler = ControlHandler::new(Arc::clone(®istry));
15618 handler
15619 .handle_control(
15620 ConnectionId::new(41),
15621 non_routable_hello_frame_with_control_ops("health-only", 306, None),
15622 )
15623 .unwrap();
15624
15625 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
15626 let response = handler
15627 .handle_control_frame(
15628 &ctx,
15629 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
15630 )
15631 .await
15632 .unwrap();
15633
15634 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
15635 assert!(parse_error(&response[0])["message"]
15636 .as_str()
15637 .unwrap()
15638 .contains("does not provide the requested target"));
15639 }
15640
15641 #[tokio::test]
15642 async fn route_open_inactive_registration_remains_target_unavailable() {
15643 let registry = Arc::new(Registry::default());
15644 let handler = ControlHandler::new(Arc::clone(®istry));
15645 handler
15646 .handle_control(
15647 ConnectionId::new(43),
15648 hello_frame("inactive", PROTOCOL_VERSION, 308),
15649 )
15650 .unwrap();
15651 assert!(registry
15652 .set_module_state_for_test("inactive", ChannelState::Closed)
15653 .unwrap());
15654
15655 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
15656 let response = handler
15657 .handle_control_frame(
15658 &ctx,
15659 route_open_frame(309, "inactive", unique_project_root("inactive")),
15660 )
15661 .await
15662 .unwrap();
15663
15664 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
15665 assert!(parse_error(&response[0])["message"]
15666 .as_str()
15667 .unwrap()
15668 .contains("is not active"));
15669 }
15670
15671 #[tokio::test]
15672 async fn late_health_reply_is_recorded_through_the_module_response_path() {
15673 let registry = Arc::new(Registry::default());
15674 let forwarding = Arc::new(ForwardingTable::default());
15675 let supervisor_handle = SupervisorHandle::new();
15676 let supervisor =
15677 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
15678 .with_forwarding(Arc::clone(&forwarding))
15679 .with_handle(supervisor_handle.clone());
15680 let module = supervisor
15681 .supervise_configured(
15682 crate::ModuleSpec {
15683 carry_containment: false,
15684 module_id: "late-health-response".to_string(),
15685 program: PathBuf::from("disabled-module"),
15686 args: Vec::new(),
15687 env: Vec::new(),
15688 reserved: false,
15689 reserved_prefixes: Vec::new(),
15690 protocol: ModuleProtocol::Subc,
15691 overlap: Default::default(),
15692 },
15693 false,
15694 )
15695 .unwrap();
15696 let handler =
15697 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15698 .with_supervisor(supervisor_handle);
15699 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
15700 handler
15701 .handle_control_frame(
15702 &module_ctx,
15703 hello_frame_with_control_ops(
15704 "late-health-response",
15705 PROTOCOL_VERSION,
15706 7,
15707 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15708 ),
15709 )
15710 .await
15711 .unwrap();
15712 let probe_started_at = Instant::now() - Duration::from_millis(80);
15713 let pending = forwarding
15714 .begin_health_probe_rpc_for(
15715 "late-health-response",
15716 MODULE_CONTROL_OP_HEALTH_CHECK,
15717 probe_started_at,
15718 Instant::now() - Duration::from_millis(1),
15719 )
15720 .unwrap();
15721 assert!(forwarding
15722 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
15723 .unwrap());
15724
15725 let responses = handler
15726 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
15727 .await
15728 .unwrap();
15729
15730 assert!(responses.is_empty());
15731 let health = module.status().unwrap().health;
15732 assert_eq!(health.late_answer_count, 1);
15733 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
15734 }
15735
15736 #[tokio::test]
15737 async fn health_probe_timeout_and_module_death_are_typed() {
15738 let registry = Arc::new(Registry::default());
15739 let forwarding = Arc::new(ForwardingTable::default());
15740 let handler =
15741 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15742 .with_health_probe_timeout(Duration::from_millis(50));
15743 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
15744 hello_via_sink(
15745 &handler,
15746 &module_ctx,
15747 &mut module_rx,
15748 hello_frame_with_control_ops(
15749 "aft",
15750 PROTOCOL_VERSION,
15751 7,
15752 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15753 ),
15754 )
15755 .await;
15756
15757 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
15758 let responses = handler
15759 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
15760 .await
15761 .unwrap();
15762 assert_eq!(responses[0].header.ty, FrameType::Error);
15763 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
15764 let _ = module_rx.try_recv();
15765
15766 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
15767 let health_handler = handler.clone();
15768 let death_task = tokio::spawn(async move {
15769 health_handler
15770 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
15771 .await
15772 .unwrap()
15773 });
15774 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15775 .await
15776 .unwrap()
15777 .unwrap();
15778 handler
15779 .cleanup_connection(module_ctx.connection_id)
15780 .unwrap();
15781 let responses = death_task.await.unwrap();
15782 assert_eq!(responses[0].header.ty, FrameType::Error);
15783 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
15784 }
15785
15786 #[test]
15787 fn hello_requires_exact_protocol_version() {
15788 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
15789 let registry = Arc::new(Registry::default());
15790 let handler = ControlHandler::new(Arc::clone(®istry));
15791 let responses = handler
15792 .handle_control(
15793 ConnectionId::new(connection),
15794 hello_frame("aft", offered, 9),
15795 )
15796 .unwrap();
15797
15798 assert_eq!(responses.len(), 1);
15799 assert_eq!(responses[0].header.ty, FrameType::Error);
15800 let error = parse_error(&responses[0]);
15801 assert_eq!(error["code"], "version_unsupported");
15802 assert!(registry.get_module("aft").unwrap().is_none());
15803 assert_eq!(registry.active_registration_count().unwrap(), 0);
15804 }
15805 }
15806
15807 #[test]
15808 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
15809 let registry = Arc::new(Registry::default());
15810 let forwarding = Arc::new(ForwardingTable::default());
15811 let handler =
15812 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15813 let module_connection = ConnectionId::new(301);
15814 let registration = registry
15815 .register_with_control_ops(
15816 manifest("aft-push", PROTOCOL_VERSION),
15817 PROTOCOL_VERSION,
15818 module_connection,
15819 module_baseline_control_ops(),
15820 )
15821 .unwrap();
15822 let (module_tx, _module_rx) = mpsc::channel(8);
15823 let endpoint = forwarding
15824 .register_module_connection(
15825 module_connection,
15826 "aft-push".to_string(),
15827 PROTOCOL_VERSION,
15828 manifest_concurrency(®istration.manifest),
15829 FrameSink::new(module_tx),
15830 )
15831 .unwrap();
15832
15833 let unknown = Frame::build(
15835 FrameType::Push,
15836 control_flags(),
15837 0,
15838 0,
15839 5,
15840 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
15841 )
15842 .unwrap();
15843 let out = handler.handle_status_update(endpoint, unknown).unwrap();
15844 assert!(
15845 out.is_empty(),
15846 "unknown push op must be ignored, got {out:?}"
15847 );
15848
15849 let malformed = Frame::build(
15851 FrameType::Push,
15852 control_flags(),
15853 0,
15854 0,
15855 6,
15856 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
15857 )
15858 .unwrap();
15859 let out = handler.handle_status_update(endpoint, malformed).unwrap();
15860 assert_eq!(out.len(), 1);
15861 assert_eq!(out[0].header.ty, FrameType::Error);
15862 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
15863 }
15864
15865 #[test]
15866 fn hello_rejected_when_connection_already_owns_client_routes() {
15867 let registry = Arc::new(Registry::default());
15868 let forwarding = Arc::new(ForwardingTable::default());
15869 let handler =
15870 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15871 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
15873 let client_connection = ConnectionId::new(202);
15874
15875 let responses = handler
15878 .handle_control(
15879 client_connection,
15880 hello_frame("aft-second", PROTOCOL_VERSION, 9),
15881 )
15882 .unwrap();
15883 assert_eq!(responses[0].header.ty, FrameType::Error);
15884 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
15885 assert!(registry.get_module("aft-second").unwrap().is_none());
15886 }
15887
15888 #[tokio::test]
15889 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
15890 let registry = Arc::new(Registry::default());
15891 let forwarding = Arc::new(ForwardingTable::default());
15892 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
15893 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
15894 hello_via_sink(
15895 &handler,
15896 &module_ctx,
15897 &mut module_rx,
15898 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
15899 )
15900 .await;
15901 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
15902 let pending = forwarding
15903 .begin_route_bind_relay_for_test(
15904 client_ctx.connection_id,
15905 client_ctx.egress.clone(),
15906 2,
15907 "alpha",
15908 )
15909 .unwrap();
15910 forwarding
15911 .complete_pending_relay(
15912 module_ctx.connection_id,
15913 pending.corr,
15914 RouteBindRelayOutcome::Accepted,
15915 )
15916 .unwrap();
15917 client_rx.try_recv().unwrap();
15918 for module_id in ["beta", "alpha"] {
15919 let replies = handler
15920 .handle_control_frame(
15921 &module_ctx,
15922 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
15923 )
15924 .await
15925 .unwrap();
15926 assert_eq!(replies.len(), 1, "second HELLO must be refused");
15927 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
15928 }
15929 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
15930 assert!(registry.get_module("beta").unwrap().is_none());
15931 assert!(matches!(
15932 forwarding
15933 .lookup_data_route(
15934 client_ctx.connection_id,
15935 pending.client_channel,
15936 pending.client_epoch,
15937 )
15938 .unwrap(),
15939 DataRoute::Client(DataRouteState::Bound(_))
15940 ));
15941 assert!(handler
15942 .hello_launch_nonces
15943 .lock()
15944 .unwrap()
15945 .presented(module_ctx.connection_id, Some("alpha-nonce")));
15946 assert!(module_rx.try_recv().is_err());
15947 }
15948
15949 #[test]
15950 fn reserved_module_hello_requires_matching_launch_nonce() {
15951 let registry = Arc::new(Registry::default());
15952 let supervisor = SupervisorHandle::new();
15953 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
15956 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15957
15958 let no_nonce = handler
15960 .handle_control(
15961 ConnectionId::new(1),
15962 hello_frame("vault", PROTOCOL_VERSION, 1),
15963 )
15964 .unwrap();
15965 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
15966 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
15967 assert!(registry.get_module("vault").unwrap().is_none());
15968
15969 let wrong = handler
15971 .handle_control(
15972 ConnectionId::new(2),
15973 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
15974 )
15975 .unwrap();
15976 assert_eq!(wrong[0].header.ty, FrameType::Error);
15977 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
15978 assert!(registry.get_module("vault").unwrap().is_none());
15979
15980 let ok = handler
15982 .handle_control(
15983 ConnectionId::new(3),
15984 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
15985 )
15986 .unwrap();
15987 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
15988 assert!(registry.get_module("vault").unwrap().is_some());
15989 }
15990
15991 #[test]
15992 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
15993 let registry = Arc::new(Registry::default());
15994 let supervisor = SupervisorHandle::new();
15995 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15996 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15997 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15998
15999 let squat = handler
16000 .handle_control(
16001 ConnectionId::new(1),
16002 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
16003 )
16004 .unwrap();
16005 assert_eq!(squat[0].header.ty, FrameType::Error);
16006 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
16007 assert!(parse_error(&squat[0])["message"]
16008 .as_str()
16009 .unwrap()
16010 .contains("fed:"));
16011
16012 let accepted_peer = handler
16013 .handle_control(
16014 ConnectionId::new(2),
16015 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
16016 )
16017 .unwrap();
16018 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
16019
16020 let accepted_short = handler
16021 .handle_control(
16022 ConnectionId::new(3),
16023 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
16024 )
16025 .unwrap();
16026 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
16027
16028 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
16029 let response = handler
16030 .handle_control(
16031 ConnectionId::new(conn),
16032 hello_frame(module_id, PROTOCOL_VERSION, conn),
16033 )
16034 .unwrap();
16035 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
16036 }
16037 }
16038
16039 #[test]
16040 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
16041 let registry = Arc::new(Registry::default());
16042 let supervisor = SupervisorHandle::new();
16043 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
16044 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
16045 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
16046 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
16047
16048 let owner_nonce = handler
16049 .handle_control(
16050 ConnectionId::new(1),
16051 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
16052 )
16053 .unwrap();
16054 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
16055 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
16056 assert!(registry.get_module("fed:special").unwrap().is_none());
16057
16058 let exact_nonce = handler
16059 .handle_control(
16060 ConnectionId::new(2),
16061 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
16062 )
16063 .unwrap();
16064 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
16065 assert!(registry.get_module("fed:special").unwrap().is_some());
16066 }
16067
16068 #[test]
16069 fn non_reserved_module_ignores_launch_nonce() {
16070 let registry = Arc::new(Registry::default());
16071 let handler = ControlHandler::new(Arc::clone(®istry));
16074 let no_nonce = handler
16075 .handle_control(
16076 ConnectionId::new(1),
16077 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
16078 )
16079 .unwrap();
16080 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
16081 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
16082
16083 let echoed_nonce = handler
16084 .handle_control(
16085 ConnectionId::new(2),
16086 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
16087 )
16088 .unwrap();
16089 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
16090 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
16091 }
16092
16093 #[test]
16094 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
16095 let handler = ControlHandler::default();
16096 let conn = ConnectionId::new(1);
16097 let malformed = Frame::build(
16098 FrameType::Hello,
16099 control_flags(),
16100 0,
16101 0,
16102 3,
16103 b"{not json".to_vec(),
16104 )
16105 .unwrap();
16106
16107 let error = handler.handle_control(conn, malformed).unwrap();
16108 assert_eq!(error[0].header.ty, FrameType::Error);
16109 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
16110
16111 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
16112 let pong = handler.handle_control(conn, ping).unwrap();
16113 assert_eq!(pong[0].header.ty, FrameType::Pong);
16114 assert_eq!(pong[0].header.corr, 4);
16115 }
16116
16117 #[test]
16118 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
16119 let registry = Arc::new(Registry::default());
16120 let handler = ControlHandler::new(Arc::clone(®istry));
16121
16122 handler
16123 .handle_control(
16124 ConnectionId::new(1),
16125 hello_frame("aft", PROTOCOL_VERSION, 1),
16126 )
16127 .unwrap();
16128 let duplicate = handler
16129 .handle_control(
16130 ConnectionId::new(2),
16131 hello_frame("aft", PROTOCOL_VERSION, 2),
16132 )
16133 .unwrap();
16134
16135 assert_eq!(duplicate[0].header.ty, FrameType::Error);
16136 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
16137 let registration = registry.get_module("aft").unwrap().unwrap();
16138 assert_eq!(registration.connection_id, ConnectionId::new(1));
16139 }
16140
16141 #[test]
16142 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
16143 let registry = Arc::new(Registry::default());
16144 let forwarding = Arc::new(ForwardingTable::default());
16145 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
16146 let handler =
16147 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16148 .with_process_liveness(process_liveness);
16149 let (ctx, route_channel, route_epoch) =
16150 bind_liveness_route(®istry, &forwarding, "aft-dead");
16151 let responses = handler
16152 .handle_route_poll(
16153 &ctx,
16154 route_poll_frame(41, PollKind::Liveness, route_channel),
16155 route_channel,
16156 route_epoch,
16157 PollKind::Liveness,
16158 )
16159 .unwrap();
16160
16161 assert_eq!(responses.len(), 1);
16162 assert_eq!(responses[0].header.ty, FrameType::Response);
16163 assert_route_poll_liveness(&responses[0], false);
16164 }
16165
16166 #[test]
16167 fn liveness_poll_without_process_source_uses_bound_route() {
16168 let registry = Arc::new(Registry::default());
16169 let forwarding = Arc::new(ForwardingTable::default());
16170 let handler =
16171 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
16172 let (ctx, route_channel, route_epoch) =
16173 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
16174 let responses = handler
16175 .handle_route_poll(
16176 &ctx,
16177 route_poll_frame(42, PollKind::Liveness, route_channel),
16178 route_channel,
16179 route_epoch,
16180 PollKind::Liveness,
16181 )
16182 .unwrap();
16183
16184 assert_route_poll_liveness(&responses[0], true);
16185 }
16186
16187 #[test]
16188 fn liveness_poll_untracked_process_source_uses_bound_route() {
16189 let registry = Arc::new(Registry::default());
16190 let forwarding = Arc::new(ForwardingTable::default());
16191 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
16192 let handler =
16193 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16194 .with_process_liveness(process_liveness);
16195 let (ctx, route_channel, route_epoch) =
16196 bind_liveness_route(®istry, &forwarding, "aft-untracked");
16197 let responses = handler
16198 .handle_route_poll(
16199 &ctx,
16200 route_poll_frame(43, PollKind::Liveness, route_channel),
16201 route_channel,
16202 route_epoch,
16203 PollKind::Liveness,
16204 )
16205 .unwrap();
16206
16207 assert_route_poll_liveness(&responses[0], true);
16208 }
16209
16210 #[tokio::test]
16211 async fn unknown_op_returns_unknown_control_op() {
16212 let handler = ControlHandler::default();
16213 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
16214 let request = Frame::build(
16215 FrameType::Request,
16216 control_flags(),
16217 0,
16218 0,
16219 55,
16220 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
16221 )
16222 .unwrap();
16223
16224 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16225
16226 assert_eq!(response.len(), 1);
16227 assert_eq!(response[0].header.ty, FrameType::Error);
16228 assert_eq!(response[0].header.corr, 55);
16229 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
16230 }
16231
16232 #[tokio::test]
16233 async fn supervisor_provenance_rejects_unknown_exact_module() {
16234 let handler = ControlHandler::default();
16235 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
16236 let request = Frame::build(
16237 FrameType::Request,
16238 control_flags(),
16239 0,
16240 0,
16241 57,
16242 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
16243 )
16244 .unwrap();
16245
16246 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16247
16248 assert_eq!(response.len(), 1);
16249 assert_eq!(response[0].header.ty, FrameType::Error);
16250 assert_eq!(response[0].header.corr, 57);
16251 let error = parse_error(&response[0]);
16252 assert_eq!(error["code"], "unknown_module");
16253 assert_eq!(error["message"], "module_id 'missing' is not supervised");
16254 }
16255
16256 #[test]
16257 fn provenance_probe_override_keeps_handler_tests_deterministic() {
16258 let expected = subc_control::RunningImageAgreement::Unavailable {
16259 reason: subc_control::RunningImageUnavailableReason::HashFailed,
16260 };
16261 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
16262 assert_eq!(handler.provenance_probe_override, Some(expected));
16263 }
16264
16265 #[test]
16266 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
16267 let verdict = reload_verdict(
16268 std::path::Path::new("/bin/new"),
16269 Some(std::path::Path::new("/bin/old")),
16270 subc_control::RunningImageAgreement::Unavailable {
16271 reason: subc_control::RunningImageUnavailableReason::HashFailed,
16272 },
16273 );
16274 assert!(matches!(
16275 verdict.path,
16276 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
16277 if configured == std::path::Path::new("/bin/new")
16278 && spawned_from == std::path::Path::new("/bin/old")
16279 ));
16280 }
16281
16282 #[test]
16283 fn launch_spec_without_running_digest_is_unavailable_not_match() {
16284 use subc_control::{
16285 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
16286 };
16287 let configured = crate::supervise::ModuleSpec {
16288 carry_containment: false,
16289 module_id: "legacy".into(),
16290 program: "/module".into(),
16291 args: Vec::new(),
16292 env: Vec::new(),
16293 reserved: false,
16294 reserved_prefixes: Vec::new(),
16295 protocol: subc_control::ModuleProtocol::Subc,
16296 overlap: Default::default(),
16297 };
16298 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
16299 assert_eq!(
16300 launch_spec_agreement(&configured, false, Some(&digest)),
16301 Agreement::Unavailable {
16302 reason: Reason::NotRunning
16303 }
16304 );
16305 assert_eq!(
16306 launch_spec_agreement(&configured, true, None),
16307 Agreement::Unavailable {
16308 reason: Reason::SpawnedSpecUnavailable
16309 }
16310 );
16311 assert_eq!(
16312 launch_spec_agreement(&configured, true, Some(&digest)),
16313 Agreement::Match
16314 );
16315 }
16316
16317 #[cfg(windows)]
16318 #[test]
16319 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
16320 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
16321 let configured = dir.join("module.exe");
16322 std::fs::write(&configured, b"executable path fixture").unwrap();
16323 let spawned = std::fs::canonicalize(&configured).unwrap();
16324 assert_ne!(
16325 configured, spawned,
16326 "fixture must exercise different path spellings"
16327 );
16328 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
16329 let lower_case = configured.to_string_lossy().to_lowercase();
16330 for spelling in [
16331 &configured,
16332 Path::new(&forward_slashes),
16333 Path::new(&lower_case),
16334 ] {
16335 let verdict = reload_verdict(
16336 spelling,
16337 Some(&spawned),
16338 subc_control::RunningImageAgreement::Unavailable {
16339 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16340 },
16341 );
16342 assert_eq!(
16343 verdict.path,
16344 ReloadPathAgreement::Match,
16345 "configured {spelling:?}, spawned {spawned:?}"
16346 );
16347 }
16348 }
16349
16350 #[cfg(windows)]
16351 #[test]
16352 fn windows_reload_path_keeps_different_existing_program_pending() {
16353 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
16354 let old = dir.join("old.exe");
16355 let new = dir.join("new.exe");
16356 std::fs::write(&old, b"same executable content").unwrap();
16357 std::fs::copy(&old, &new).unwrap();
16358 let spawned = std::fs::canonicalize(&old).unwrap();
16359 let image = subc_control::RunningImageAgreement::Unavailable {
16360 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16361 };
16362 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
16363 assert!(
16364 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
16365 if configured == new && spawned_from == spawned)
16366 );
16367 let missing = dir.join("missing.exe");
16368 assert!(matches!(
16369 reload_verdict(&missing, Some(&spawned), image).path,
16370 ReloadPathAgreement::Mismatch { .. }
16371 ));
16372 }
16373
16374 #[test]
16375 fn reload_verdict_detects_replaced_image_at_same_path() {
16376 let image = subc_control::RunningImageAgreement::Mismatch {
16377 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
16378 digest: "old".into(),
16379 },
16380 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
16381 digest: "new".into(),
16382 },
16383 };
16384 let verdict = reload_verdict(
16385 std::path::Path::new("/bin/same"),
16386 Some(std::path::Path::new("/bin/same")),
16387 image.clone(),
16388 );
16389 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16390 assert_eq!(verdict.image, image);
16391 }
16392
16393 #[test]
16394 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
16395 let image = subc_control::RunningImageAgreement::Unavailable {
16396 reason: subc_control::RunningImageUnavailableReason::NotRunning,
16397 };
16398 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
16399 assert_eq!(
16400 verdict.path,
16401 subc_control::ReloadPathAgreement::Unavailable {
16402 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
16403 }
16404 );
16405 assert_eq!(verdict.image, image);
16406
16407 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
16408 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16409 };
16410 let verdict = reload_verdict(
16411 std::path::Path::new("/bin/same"),
16412 Some(std::path::Path::new("/bin/same")),
16413 unconfirmed.clone(),
16414 );
16415 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16416 assert_eq!(verdict.image, unconfirmed);
16417 }
16418
16419 #[test]
16420 fn reload_verdict_preserves_each_image_unavailability_reason() {
16421 use subc_control::RunningImageUnavailableReason as Reason;
16422
16423 for reason in [
16424 Reason::NotRunning,
16425 Reason::UnsupportedPlatform,
16426 Reason::RunningExecutableUnreadable,
16427 Reason::SpawnedPathUnreadable,
16428 Reason::HashFailed,
16429 Reason::ProcessIdentityUnconfirmed,
16430 Reason::Unknown("future_probe_reason".to_string()),
16431 ] {
16432 let image = subc_control::RunningImageAgreement::Unavailable {
16433 reason: reason.clone(),
16434 };
16435 let verdict = reload_verdict(
16436 std::path::Path::new("/bin/same"),
16437 Some(std::path::Path::new("/bin/same")),
16438 image.clone(),
16439 );
16440 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16441 assert_eq!(verdict.image, image, "{reason:?}");
16442 }
16443 }
16444
16445 #[tokio::test]
16446 async fn malformed_control_bodies_return_invalid_control_body() {
16447 let handler = ControlHandler::default();
16448 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
16449
16450 for (corr, body) in [
16451 (56, br#"{"route_channel":1}"#.as_slice()),
16452 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
16453 (
16454 58,
16455 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
16456 ),
16457 ] {
16458 let request = Frame::build(
16459 FrameType::Request,
16460 control_flags(),
16461 0,
16462 0,
16463 corr,
16464 body.to_vec(),
16465 )
16466 .unwrap();
16467 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16468
16469 assert_eq!(response.len(), 1);
16470 assert_eq!(response[0].header.ty, FrameType::Error);
16471 assert_eq!(response[0].header.corr, corr);
16472 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
16473 }
16474 }
16475
16476 #[tokio::test]
16477 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
16478 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16479 let registry = Arc::new(Registry::default());
16480 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
16481 let router = Router::with_control_handler(Arc::clone(&control));
16482 let connection = router.begin_connection();
16483 let (ctx, mut rx) = route_ctx(connection.id());
16484
16485 router
16486 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
16487 .await
16488 .unwrap();
16489 let response = rx.recv().await.unwrap();
16490 let ack = parse_ack(&response);
16491 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16492 let channel = 1;
16493
16494 let goodbye =
16495 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
16496 router.route_for_connection(&ctx, goodbye).await.unwrap();
16497 assert!(rx.try_recv().is_err());
16498 assert!(registry.get_module("aft").unwrap().is_none());
16499
16500 router
16501 .route_for_connection(&ctx, channel_request(channel, 13))
16502 .await
16503 .unwrap();
16504 let error_frame = rx.recv().await.unwrap();
16505 assert_eq!(error_frame.header.ty, FrameType::Error);
16506 assert_eq!(error_frame.header.channel, channel);
16507 let captured = crate::router::test_log::captured_logs(&logs);
16508 assert_eq!(
16509 captured
16510 .lines()
16511 .filter(|line| {
16512 line.contains("module_id=aft")
16513 && line.contains("reason=explicit_goodbye")
16514 && line.contains("module registration ended")
16515 })
16516 .count(),
16517 1,
16518 "unexpected GOODBYE registry log: {captured}"
16519 );
16520 }
16521
16522 #[tokio::test]
16523 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
16524 let registry = Arc::new(Registry::default());
16525 let handler = ControlHandler::new(registry).with_capability_config(
16526 [("prov".to_string(), true), ("cons".to_string(), true)],
16527 BTreeMap::new(),
16528 );
16529 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
16530 register_capability_manifest(
16531 &handler,
16532 &provider_ctx,
16533 &mut provider_rx,
16534 capability_manifest("prov", &["thing/v1"], &[]),
16535 1,
16536 )
16537 .await;
16538 let mut consumer = capability_manifest("cons", &[], &[]);
16539 consumer.capabilities.as_mut().unwrap().requires.push(
16540 subc_protocol::manifest::CapabilityRequirement {
16541 capability: "thing/v1".to_string(),
16542 need: subc_protocol::manifest::CapabilityNeed::Required,
16543 },
16544 );
16545 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
16546 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
16547 assert_eq!(
16548 handler.capability_evaluator.verdict("cons", "thing/v1"),
16549 Some(CapabilityVerdict::Provided)
16550 );
16551 let (mut client_rx, _) = open_route_for_capability_test(
16552 &handler,
16553 &provider_ctx,
16554 &mut provider_rx,
16555 703,
16556 3,
16557 "prov",
16558 None,
16559 )
16560 .await;
16561 let goodbye =
16562 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
16563 handler
16564 .handle_control_frame(&provider_ctx, goodbye)
16565 .await
16566 .unwrap();
16567 assert_eq!(
16568 handler.capability_evaluator.verdict("cons", "thing/v1"),
16569 Some(CapabilityVerdict::NeverProvided)
16570 );
16571 let closed = client_rx
16572 .try_recv()
16573 .expect("GOODBYE pushes route.closed before route GOODBYE");
16574 assert!(
16575 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
16576 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
16577 );
16578 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
16579 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
16580 }
16581
16582 struct RouteClosedFixture {
16583 handler: ControlHandler,
16584 #[cfg(unix)]
16585 supervisor: Supervisor,
16586 module_ctx: RouteCtx,
16587 #[cfg(unix)]
16588 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16589 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16590 route: Arc<crate::forwarding::RouteBinding>,
16591 }
16592
16593 async fn route_closed_fixture() -> RouteClosedFixture {
16594 let registry = Arc::new(Registry::default());
16595 let handler = ControlHandler::new(Arc::clone(®istry));
16596 #[cfg(unix)]
16597 let supervisor = Supervisor::new(registry, RestartPolicy::default())
16598 .with_forwarding(Arc::clone(&handler.forwarding));
16599 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
16600 register_capability_manifest(
16601 &handler,
16602 &module_ctx,
16603 &mut module_rx,
16604 capability_manifest("prov", &[], &[]),
16605 1,
16606 )
16607 .await;
16608 let (client_rx, bind) = open_route_for_capability_test(
16609 &handler,
16610 &module_ctx,
16611 &mut module_rx,
16612 702,
16613 2,
16614 "prov",
16615 None,
16616 )
16617 .await;
16618 let ModuleControlRequest::RouteBind {
16619 route_channel: channel,
16620 epoch: route_epoch,
16621 ..
16622 } = bind
16623 else {
16624 panic!("expected route.bind");
16625 };
16626 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
16627 handler
16628 .forwarding
16629 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
16630 .unwrap()
16631 else {
16632 panic!("the acknowledged route must be bound");
16633 };
16634 RouteClosedFixture {
16635 handler,
16636 #[cfg(unix)]
16637 supervisor,
16638 module_ctx,
16639 #[cfg(unix)]
16640 module_rx,
16641 client_rx,
16642 route,
16643 }
16644 }
16645
16646 impl RouteClosedFixture {
16647 #[cfg(unix)]
16648 async fn cut(&mut self) {
16649 let notices = async {
16650 let mut draining = self.module_rx.recv().await.unwrap();
16651 assert!(matches!(
16652 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
16653 &draining.body
16654 )
16655 .unwrap(),
16656 subc_protocol::session::ModuleControlCommand::Draining {
16657 reason: RouteCloseReason::Disable,
16658 ..
16659 }
16660 ));
16661 draining.flushed.take().unwrap().send(()).unwrap();
16662 let mut closing = self.client_rx.recv().await.unwrap();
16663 assert!(matches!(
16664 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
16665 ClientControlPush::RouteClosing {
16666 reason: RouteCloseReason::Restart,
16667 ..
16668 }
16669 ));
16670 closing.flushed.take().unwrap().send(()).unwrap();
16671 };
16672 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
16673 result.unwrap();
16674 }
16675
16676 fn disconnect(&mut self) -> ClientControlPush {
16677 self.handler
16678 .cleanup_connection(self.module_ctx.connection_id)
16679 .unwrap();
16680 let closed = self.client_rx.try_recv().unwrap();
16681 assert_eq!(closed.header.ty, FrameType::Push);
16682 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
16683 assert!(matches!(
16684 &push,
16685 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
16686 if channels == &[self.route.client_channel]
16687 ));
16688 assert_eq!(
16689 self.client_rx.try_recv().unwrap().header.ty,
16690 FrameType::Goodbye
16691 );
16692 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
16693 push
16694 }
16695 }
16696
16697 #[cfg(unix)]
16698 #[tokio::test(start_paused = true)]
16699 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
16700 let mut fixture = route_closed_fixture().await;
16701 fixture.cut().await;
16702 assert!(matches!(
16703 fixture.disconnect(),
16704 ClientControlPush::RouteClosed {
16705 reason: RouteCloseReason::Restart,
16706 drained: true,
16707 ..
16708 }
16709 ));
16710 }
16711
16712 #[cfg(unix)]
16713 #[tokio::test(start_paused = true)]
16714 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
16715 let mut fixture = route_closed_fixture().await;
16716 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
16717 let started = tokio::time::Instant::now();
16718 fixture.cut().await;
16719 assert!(started.elapsed() >= Duration::from_secs(2));
16720 assert_eq!(fixture.route.flow.in_flight(), 1);
16721 assert!(matches!(
16722 fixture.disconnect(),
16723 ClientControlPush::RouteClosed {
16724 reason: RouteCloseReason::Restart,
16725 drained: false,
16726 ..
16727 }
16728 ));
16729 }
16730
16731 #[tokio::test]
16732 async fn module_connection_crash_reports_crash_and_not_drained() {
16733 let mut fixture = route_closed_fixture().await;
16734 assert!(matches!(
16738 fixture.disconnect(),
16739 ClientControlPush::RouteClosed {
16740 reason: RouteCloseReason::Crash,
16741 drained: false,
16742 ..
16743 }
16744 ));
16745 }
16746
16747 #[tokio::test]
16748 async fn dropping_router_connection_releases_registration() {
16749 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16750 let registry = Arc::new(Registry::default());
16751 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
16752 let router = Router::with_control_handler(Arc::clone(&control));
16753 let connection = router.begin_connection();
16754 let connection_id = connection.id();
16755 let (ctx, mut rx) = route_ctx(connection_id);
16756
16757 router
16758 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
16759 .await
16760 .unwrap();
16761 let response = rx.recv().await.unwrap();
16762 let ack = parse_ack(&response);
16763 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16764 assert!(registry.get_module("aft").unwrap().is_some());
16765
16766 drop(connection);
16767
16768 assert!(registry.get_module("aft").unwrap().is_none());
16769 assert_eq!(registry.active_registration_count().unwrap(), 0);
16770
16771 control
16772 .cleanup_connection(ConnectionId::new(u64::MAX))
16773 .unwrap();
16774 let captured = crate::router::test_log::captured_logs(&logs);
16775 println!("captured registry lifecycle logs:\n{captured}");
16776 let events: Vec<_> = captured
16777 .lines()
16778 .filter(|line| {
16779 line.contains("module registered module_id=aft ")
16780 || line.contains("module registration ended")
16781 })
16782 .collect();
16783 assert_eq!(
16784 events.len(),
16785 2,
16786 "unexpected registry lifecycle logs: {captured}"
16787 );
16788 assert!(events[0].contains("module registered module_id=aft "));
16789 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
16790 assert!(events[1].contains(&format!(
16791 "module_id=aft connection_id={}",
16792 connection_id.get()
16793 )));
16794 assert!(events[1].contains("reason=connection_closed"));
16795 assert!(events[1].contains("module registration ended"));
16796 assert_eq!(
16797 captured
16798 .lines()
16799 .filter(|line| line.contains("module registered module_id=aft "))
16800 .count(),
16801 1,
16802 "legacy registration admission line must appear once: {captured}"
16803 );
16804 }
16805
16806 #[tokio::test]
16807 async fn hello_registration_keeps_legacy_module_registered_line_once() {
16808 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16809 let registry = Arc::new(Registry::default());
16810 let control = ControlHandler::new(Arc::clone(®istry));
16811 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
16812 hello_via_sink(
16813 &control,
16814 &ctx,
16815 &mut rx,
16816 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
16817 )
16818 .await;
16819
16820 let captured = crate::router::test_log::captured_logs(&logs);
16821 let admissions: Vec<_> = captured
16822 .lines()
16823 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
16824 .collect();
16825 assert_eq!(
16826 admissions.len(),
16827 1,
16828 "legacy admission line must remain exactly once: {captured}"
16829 );
16830 assert!(admissions[0].contains("routable_provider=true"));
16831 assert!(admissions[0].contains("connection_id=777"));
16832 }
16833
16834 fn capability_manifest(
16835 module_id: &str,
16836 provides: &[&str],
16837 must_never_reach: &[&str],
16838 ) -> ModuleManifest {
16839 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
16840 manifest.capabilities = Some(CapabilityDeclarations {
16841 provides: provides
16842 .iter()
16843 .map(|capability| (*capability).to_string())
16844 .collect(),
16845 requires: Vec::new(),
16846 must_never_reach: must_never_reach
16847 .iter()
16848 .map(|capability| (*capability).to_string())
16849 .collect(),
16850 });
16851 manifest
16852 }
16853
16854 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
16855 Frame::build(
16856 FrameType::Hello,
16857 control_flags(),
16858 0,
16859 0,
16860 corr,
16861 serde_json::to_vec(&ModuleHelloBody {
16862 protocol_ver: manifest.protocol_ver,
16863 manifest,
16864 control_ops: None,
16865 launch_nonce: None,
16866 })
16867 .expect("capability test HELLO serializes"),
16868 )
16869 .expect("capability test HELLO frame builds")
16870 }
16871
16872 fn catalog_update_with_capabilities_frame(
16873 corr: u64,
16874 capabilities: CapabilityDeclarations,
16875 ) -> Frame {
16876 Frame::build(
16877 FrameType::Request,
16878 control_flags(),
16879 0,
16880 0,
16881 corr,
16882 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16883 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
16884 capabilities: Some(capabilities),
16885 ready: None,
16886 })
16887 .expect("capability catalog.update serializes"),
16888 )
16889 .expect("capability catalog.update frame builds")
16890 }
16891
16892 async fn register_capability_manifest(
16893 handler: &ControlHandler,
16894 ctx: &RouteCtx,
16895 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16896 manifest: ModuleManifest,
16897 corr: u64,
16898 ) {
16899 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
16900 }
16901
16902 async fn open_route_for_capability_test(
16903 handler: &ControlHandler,
16904 target_ctx: &RouteCtx,
16905 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16906 client_connection_id: u64,
16907 corr: u64,
16908 target_module_id: &str,
16909 consumer_identity: Option<ConsumerIdentity>,
16910 ) -> (
16911 mpsc::Receiver<crate::router::OutboundFrame>,
16912 ModuleControlRequest,
16913 ) {
16914 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
16915 let route_handler = handler.clone();
16916 let target_module_id = target_module_id.to_string();
16917 let route_task = tokio::spawn(async move {
16918 route_handler
16919 .handle_control_frame(
16920 &client_ctx,
16921 route_open_frame_with_admission_facts(
16922 corr,
16923 &target_module_id,
16924 unique_project_root("admission-facts"),
16925 consumer_identity,
16926 None,
16927 ),
16928 )
16929 .await
16930 .expect("capability test route.open succeeds")
16931 });
16932 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
16933 .await
16934 .expect("capability test route.open must reach route.bind")
16935 .expect("target control receiver stays open");
16936 let bind_request: ModuleControlRequest =
16937 serde_json::from_slice(&bind.body).expect("route.bind decodes");
16938 handler
16939 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
16940 .await
16941 .expect("capability test route.bind ACK succeeds");
16942 assert!(route_task.await.expect("route.open task joins").is_empty());
16943 let opened = client_rx
16944 .recv()
16945 .await
16946 .expect("successful route.open publishes a response");
16947 assert!(matches!(
16948 serde_json::from_slice::<ClientControlResponse>(&opened.body),
16949 Ok(ClientControlResponse::RouteOpen { .. })
16950 ));
16951 (client_rx, bind_request)
16952 }
16953
16954 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
16955 assert_eq!(frame.header.ty, FrameType::Push);
16956 assert_eq!(frame.header.channel, 0);
16957 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
16958 .expect("route.closed control push decodes");
16959 let ClientControlPush::RouteClosed { channels, .. } = &push else {
16960 panic!("expected route.closed");
16961 };
16962 assert_eq!(channels.len(), 1, "exactly one violating route closed");
16963 let closed_channel = channels[0];
16964 let channels = channels.clone();
16965 assert_eq!(
16966 push,
16967 ClientControlPush::RouteClosed {
16968 module_id: target_module_id.to_string(),
16969 channels,
16970 reason: RouteCloseReason::CapabilityDenied,
16971 drained: false,
16972 abandoned: 0,
16973 excluded_subscriptions: 0,
16974 terminal: Some(false),
16975 }
16976 );
16977 closed_channel
16978 }
16979
16980 #[tokio::test]
16981 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
16982 let registry = Arc::new(Registry::default());
16983 let forwarding = Arc::new(ForwardingTable::default());
16984 let supervisor = SupervisorHandle::new();
16985 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16986 let handler =
16987 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16988 .with_supervisor(supervisor);
16989 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
16990 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
16991 register_capability_manifest(
16992 &handler,
16993 &target_ctx,
16994 &mut target_rx,
16995 capability_manifest("target", &["credentials-provider/v1"], &[]),
16996 1,
16997 )
16998 .await;
16999 register_capability_manifest(
17000 &handler,
17001 &opener_ctx,
17002 &mut opener_rx,
17003 capability_manifest("opener", &[], &["credentials-provider/v1"]),
17004 2,
17005 )
17006 .await;
17007
17008 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
17009 let replies = handler
17010 .handle_control_frame(
17011 &client_ctx,
17012 route_open_frame_with_admission_facts(
17013 3,
17014 "target",
17015 unique_project_root("admission-facts"),
17016 Some(ConsumerIdentity {
17017 module_id: "opener".to_string(),
17018 launch_nonce: "opener-nonce".to_string(),
17019 }),
17020 None,
17021 ),
17022 )
17023 .await
17024 .expect("denied route.open returns a typed frame");
17025 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
17026 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17027 assert!(
17028 target_rx.try_recv().is_err(),
17029 "forbidden route.open must not relay route.bind"
17030 );
17031 }
17032
17033 #[tokio::test]
17034 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
17035 let registry = Arc::new(Registry::default());
17036 let forwarding = Arc::new(ForwardingTable::default());
17037 let supervisor = SupervisorHandle::new();
17038 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17039 let handler =
17040 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17041 .with_supervisor(supervisor);
17042 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
17043 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
17044 register_capability_manifest(
17045 &handler,
17046 &target_ctx,
17047 &mut target_rx,
17048 capability_manifest("target", &["credentials-provider/v1"], &[]),
17049 1,
17050 )
17051 .await;
17052 register_capability_manifest(
17053 &handler,
17054 &old_opener_ctx,
17055 &mut old_opener_rx,
17056 capability_manifest("opener", &[], &[]),
17057 2,
17058 )
17059 .await;
17060 let (mut client_rx, _) = open_route_for_capability_test(
17061 &handler,
17062 &target_ctx,
17063 &mut target_rx,
17064 712,
17065 3,
17066 "target",
17067 Some(ConsumerIdentity {
17068 module_id: "opener".to_string(),
17069 launch_nonce: "opener-nonce".to_string(),
17070 }),
17071 )
17072 .await;
17073 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17074
17075 handler
17076 .cleanup_connection(old_opener_ctx.connection_id)
17077 .expect("old opener registration cleans up");
17078 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
17079 register_capability_manifest(
17080 &handler,
17081 &new_opener_ctx,
17082 &mut new_opener_rx,
17083 capability_manifest("opener", &[], &["credentials-provider/v1"]),
17084 4,
17085 )
17086 .await;
17087
17088 let closed_channel = assert_capability_denied_push(
17089 client_rx
17090 .try_recv()
17091 .expect("HELLO deny addition must emit route.closed")
17092 .frame,
17093 "target",
17094 );
17095 let goodbye = client_rx
17096 .try_recv()
17097 .expect("HELLO deny addition must send the client a route GOODBYE")
17098 .frame;
17099 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17100 assert_eq!(goodbye.header.channel, closed_channel);
17101 assert_ne!(goodbye.header.channel, 0);
17102 assert_ne!(goodbye.header.epoch, 0);
17103 assert!(client_rx.try_recv().is_err());
17104 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17105 assert!(matches!(
17106 target_rx.try_recv(),
17107 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
17108 ));
17109 }
17110
17111 #[tokio::test]
17112 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
17113 let registry = Arc::new(Registry::default());
17114 let forwarding = Arc::new(ForwardingTable::default());
17115 let supervisor = SupervisorHandle::new();
17116 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17117 let handler =
17118 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17119 .with_supervisor(supervisor);
17120 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
17121 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
17122 register_capability_manifest(
17123 &handler,
17124 &target_ctx,
17125 &mut target_rx,
17126 capability_manifest("target", &[], &[]),
17127 1,
17128 )
17129 .await;
17130 register_capability_manifest(
17131 &handler,
17132 &opener_ctx,
17133 &mut opener_rx,
17134 capability_manifest("opener", &[], &["credentials-provider/v1"]),
17135 2,
17136 )
17137 .await;
17138 let (mut client_rx, _) = open_route_for_capability_test(
17139 &handler,
17140 &target_ctx,
17141 &mut target_rx,
17142 722,
17143 3,
17144 "target",
17145 Some(ConsumerIdentity {
17146 module_id: "opener".to_string(),
17147 launch_nonce: "opener-nonce".to_string(),
17148 }),
17149 )
17150 .await;
17151 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17152
17153 let replies = handler
17154 .handle_control_frame(
17155 &target_ctx,
17156 catalog_update_with_capabilities_frame(
17157 4,
17158 CapabilityDeclarations {
17159 provides: vec!["credentials-provider/v1".to_string()],
17160 requires: Vec::new(),
17161 must_never_reach: Vec::new(),
17162 },
17163 ),
17164 )
17165 .await
17166 .expect("claim catalog.update succeeds");
17167 assert!(matches!(
17168 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
17169 Ok(ModuleControlResponseToModule::CatalogUpdate {})
17170 ));
17171 let closed_channel = assert_capability_denied_push(
17172 client_rx
17173 .try_recv()
17174 .expect("claim addition must emit route.closed")
17175 .frame,
17176 "target",
17177 );
17178 let goodbye = client_rx
17179 .try_recv()
17180 .expect("claim addition must send the client a route GOODBYE")
17181 .frame;
17182 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17183 assert_eq!(goodbye.header.channel, closed_channel);
17184 assert_ne!(goodbye.header.channel, 0);
17185 assert_ne!(goodbye.header.epoch, 0);
17186 assert!(client_rx.try_recv().is_err());
17187 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17188 assert!(matches!(
17189 target_rx.try_recv(),
17190 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
17191 ));
17192 }
17193
17194 #[tokio::test]
17195 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
17196 let registry = Arc::new(Registry::default());
17197 let forwarding = Arc::new(ForwardingTable::default());
17198 let supervisor = SupervisorHandle::new();
17199 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17200 let handler =
17201 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17202 .with_supervisor(supervisor);
17203 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
17204 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
17205 register_capability_manifest(
17206 &handler,
17207 &target_ctx,
17208 &mut target_rx,
17209 capability_manifest("target", &["credentials-provider/v1"], &[]),
17210 1,
17211 )
17212 .await;
17213 register_capability_manifest(
17214 &handler,
17215 &opener_ctx,
17216 &mut opener_rx,
17217 capability_manifest("opener", &[], &[]),
17218 2,
17219 )
17220 .await;
17221 let (mut client_rx, _) = open_route_for_capability_test(
17222 &handler,
17223 &target_ctx,
17224 &mut target_rx,
17225 732,
17226 3,
17227 "target",
17228 Some(ConsumerIdentity {
17229 module_id: "opener".to_string(),
17230 launch_nonce: "opener-nonce".to_string(),
17231 }),
17232 )
17233 .await;
17234 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17235
17236 handler
17237 .handle_control_frame(
17238 &target_ctx,
17239 catalog_update_with_capabilities_frame(
17240 4,
17241 CapabilityDeclarations {
17242 provides: Vec::new(),
17243 requires: Vec::new(),
17244 must_never_reach: Vec::new(),
17245 },
17246 ),
17247 )
17248 .await
17249 .expect("claim removal catalog.update succeeds");
17250 assert_eq!(
17251 forwarding.active_binding_count().unwrap(),
17252 1,
17253 "removing an attested target claim must leave the route census unchanged"
17254 );
17255 assert!(
17256 client_rx.try_recv().is_err(),
17257 "claim removal must not emit route.closed or a client route GOODBYE"
17258 );
17259 assert!(
17260 target_rx.try_recv().is_err(),
17261 "claim removal must not send the target a route GOODBYE"
17262 );
17263 }
17264
17265 #[tokio::test]
17268 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
17269 let registry = Arc::new(Registry::default());
17270 let forwarding = Arc::new(ForwardingTable::default());
17271 let supervisor = SupervisorHandle::new();
17272 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17273 let handler =
17274 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17275 .with_supervisor(supervisor);
17276 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
17277 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
17278 register_capability_manifest(
17279 &handler,
17280 &target_ctx,
17281 &mut target_rx,
17282 capability_manifest("target", &["credentials-provider/v1"], &[]),
17283 1,
17284 )
17285 .await;
17286 register_capability_manifest(
17287 &handler,
17288 &opener_ctx,
17289 &mut opener_rx,
17290 capability_manifest("opener", &[], &["credentials-provider/v1"]),
17291 2,
17292 )
17293 .await;
17294
17295 let (_client_rx, bind) = open_route_for_capability_test(
17296 &handler,
17297 &target_ctx,
17298 &mut target_rx,
17299 742,
17300 3,
17301 "target",
17302 None,
17303 )
17304 .await;
17305 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
17306 panic!("direct scope-honesty route must bind");
17307 };
17308 assert_eq!(principal, Some(Principal::Direct));
17309 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17310 }
17311
17312 #[tokio::test]
17315 async fn must_never_reach_self_route_is_capability_forbidden() {
17316 let registry = Arc::new(Registry::default());
17317 let forwarding = Arc::new(ForwardingTable::default());
17318 let supervisor = SupervisorHandle::new();
17319 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
17320 let handler =
17321 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17322 .with_supervisor(supervisor);
17323 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
17324 register_capability_manifest(
17325 &handler,
17326 &self_ctx,
17327 &mut self_rx,
17328 capability_manifest(
17329 "self-provider",
17330 &["credentials-provider/v1"],
17331 &["credentials-provider/v1"],
17332 ),
17333 1,
17334 )
17335 .await;
17336
17337 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
17338 let replies = handler
17339 .handle_control_frame(
17340 &client_ctx,
17341 route_open_frame_with_admission_facts(
17342 2,
17343 "self-provider",
17344 unique_project_root("admission-facts"),
17345 Some(ConsumerIdentity {
17346 module_id: "self-provider".to_string(),
17347 launch_nonce: "self-nonce".to_string(),
17348 }),
17349 None,
17350 ),
17351 )
17352 .await
17353 .expect("self-route refusal returns a typed frame");
17354 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
17355 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17356 assert!(
17357 self_rx.try_recv().is_err(),
17358 "self denial must not relay route.bind"
17359 );
17360 }
17361
17362 #[test]
17363 fn unsupported_channel_zero_frame_returns_error() {
17364 let handler = ControlHandler::default();
17365 let request = Frame::build(
17366 FrameType::Request,
17367 control_flags(),
17368 0,
17369 0,
17370 21,
17371 b"opaque".to_vec(),
17372 )
17373 .unwrap();
17374
17375 let response = handler
17376 .handle_control(ConnectionId::new(1), request)
17377 .unwrap();
17378
17379 assert_eq!(response[0].header.ty, FrameType::Error);
17380 assert_eq!(
17381 parse_error(&response[0])["code"],
17382 "unsupported_control_frame"
17383 );
17384 }
17385
17386 mod swap {
17391 use super::*;
17392
17393 const INCUMBENT: ConnectionId = ConnectionId::new(30);
17394 const CANDIDATE: ConnectionId = ConnectionId::new(40);
17395
17396 struct Swap {
17397 registry: Arc<Registry>,
17398 forwarding: Arc<ForwardingTable>,
17399 handler: ControlHandler,
17400 incumbent_ctx: RouteCtx,
17401 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17402 candidate_ctx: RouteCtx,
17403 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17404 }
17405
17406 async fn swap_with_incumbent() -> Swap {
17407 let registry = Arc::new(Registry::default());
17408 let forwarding = Arc::new(ForwardingTable::default());
17409 let handler =
17410 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17411 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
17412 hello_via_sink(
17413 &handler,
17414 &incumbent_ctx,
17415 &mut incumbent_rx,
17416 hello_frame("aft", PROTOCOL_VERSION, 7),
17417 )
17418 .await;
17419 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
17420 Swap {
17421 registry,
17422 forwarding,
17423 handler,
17424 incumbent_ctx,
17425 incumbent_rx,
17426 candidate_ctx,
17427 candidate_rx,
17428 }
17429 }
17430
17431 fn register_candidate(swap: &Swap, ready: Option<bool>) {
17432 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
17433 candidate_manifest.ready = ready;
17434 let registration = swap
17435 .registry
17436 .register_candidate_with_control_ops(
17437 candidate_manifest,
17438 PROTOCOL_VERSION,
17439 CANDIDATE,
17440 module_baseline_control_ops(),
17441 )
17442 .unwrap();
17443 swap.forwarding
17444 .register_candidate_module_connection(
17445 CANDIDATE,
17446 "aft".to_string(),
17447 PROTOCOL_VERSION,
17448 manifest_concurrency(®istration.manifest),
17449 swap.candidate_ctx.egress.clone(),
17450 )
17451 .unwrap();
17452 }
17453
17454 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
17455 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
17456 swap.registry.promote_candidate("aft").unwrap().unwrap();
17457 cutover.incumbent.unwrap()
17458 }
17459
17460 #[tokio::test]
17469 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
17470 let mut swap = swap_with_incumbent().await;
17471 let hold = Arc::new(std::sync::Barrier::new(2));
17472 let handler = swap
17473 .handler
17474 .clone()
17475 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
17476 handler
17477 .supervisor
17478 .open_swap("aft", "candidate-nonce".to_string());
17479 let hello = {
17480 let handler = handler.clone();
17481 let candidate_ctx = swap.candidate_ctx.clone();
17482 std::thread::spawn(move || {
17483 handler
17484 .handle_hello(
17485 candidate_ctx.connection_id,
17486 Some(candidate_ctx.egress.clone()),
17487 hello_frame_with_nonce(
17488 "aft",
17489 PROTOCOL_VERSION,
17490 8,
17491 Some("candidate-nonce"),
17492 ),
17493 )
17494 .unwrap()
17495 })
17496 };
17497
17498 hold.wait();
17499 assert!(swap
17502 .registry
17503 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
17504 .unwrap()
17505 .is_some());
17506 assert!(swap
17507 .registry
17508 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
17509 .unwrap()
17510 .is_none());
17511 assert!(swap
17512 .forwarding
17513 .module_endpoint_for_connection(CANDIDATE)
17514 .unwrap()
17515 .is_none());
17516
17517 hold.wait();
17518 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
17519 assert_eq!(
17520 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
17521 FrameType::HelloAck
17522 );
17523 let published = swap
17524 .registry
17525 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
17526 .unwrap()
17527 .expect("candidate is published once its HELLO completes");
17528 assert_eq!(published.connection_id, CANDIDATE);
17529 assert!(swap
17530 .forwarding
17531 .module_endpoint_for_connection(CANDIDATE)
17532 .unwrap()
17533 .is_some());
17534 }
17535
17536 fn keyed_total(counters: &Value, key: &str) -> u64 {
17537 counters[key]
17538 .as_object()
17539 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
17540 .unwrap_or(0)
17541 }
17542
17543 #[tokio::test]
17544 async fn replacement_logs_old_end_and_new_admission_once() {
17545 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
17546 let mut swap = swap_with_incumbent().await;
17547 swap.handler
17548 .supervisor
17549 .open_swap("aft", "candidate-nonce".to_string());
17550 hello_via_sink(
17551 &swap.handler,
17552 &swap.candidate_ctx,
17553 &mut swap.candidate_rx,
17554 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
17555 )
17556 .await;
17557 cutover(&swap);
17558 swap.handler.cleanup_connection(INCUMBENT).unwrap();
17559
17560 let captured = crate::router::test_log::captured_logs(&logs);
17561 println!("captured replacement registry logs:\n{captured}");
17562 let new_admissions: Vec<_> = captured
17563 .lines()
17564 .filter(|line| {
17565 line.contains("connection_id=40")
17566 && line.contains("swap candidate registered; not routable until cutover")
17567 })
17568 .collect();
17569 assert_eq!(
17570 new_admissions.len(),
17571 1,
17572 "unexpected admission logs: {captured}"
17573 );
17574 assert!(new_admissions[0].contains("module_id=aft"));
17575 assert!(new_admissions[0].contains("connection_id=40"));
17576 assert!(
17577 new_admissions[0].contains("swap candidate registered; not routable until cutover")
17578 );
17579 assert!(new_admissions[0].contains("ready=true"));
17580
17581 let old_admissions: Vec<_> = captured
17582 .lines()
17583 .filter(|line| {
17584 line.contains("module registered module_id=aft ")
17585 && line.contains("connection_id=30")
17586 })
17587 .collect();
17588 assert_eq!(
17589 old_admissions.len(),
17590 1,
17591 "unexpected incumbent admission: {captured}"
17592 );
17593
17594 let old_ends: Vec<_> = captured
17595 .lines()
17596 .filter(|line| {
17597 line.contains("connection_id=30") && line.contains("module registration ended")
17598 })
17599 .collect();
17600 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
17601 assert!(old_ends[0].contains("module_id=aft"));
17602 assert!(old_ends[0].contains("connection_id=30"));
17603 assert!(old_ends[0].contains("reason=replaced"));
17604 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
17605 assert_eq!(
17606 captured
17607 .lines()
17608 .filter(|line| line.contains("module registration promoted"))
17609 .count(),
17610 1,
17611 "unexpected promotion logs: {captured}"
17612 );
17613 }
17614
17615 #[tokio::test]
17621 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
17622 let mut swap = swap_with_incumbent().await;
17623 let handler = swap.handler.clone();
17624
17625 let cotenant = ConnectionId::new(31);
17627 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
17628 let (cotenant_task, cotenant_bind) = relay_route_open(
17629 &handler,
17630 cotenant,
17631 &cotenant_ctx.egress,
17632 &mut swap.incumbent_rx,
17633 100,
17634 "aft",
17635 "swap-cotenant",
17636 )
17637 .await;
17638 handler
17639 .handle_control_frame(
17640 &swap.incumbent_ctx,
17641 route_bind_ack(cotenant_bind.header.corr),
17642 )
17643 .await
17644 .unwrap();
17645 assert!(cotenant_task.await.unwrap().is_empty());
17646 let (cotenant_channel, cotenant_epoch) =
17647 published_route(&cotenant_rx.recv().await.unwrap());
17648
17649 let caller = ConnectionId::new(32);
17651 let (caller_ctx, mut caller_rx) = route_ctx(caller);
17652 let (caller_task, caller_bind) = relay_route_open(
17653 &handler,
17654 caller,
17655 &caller_ctx.egress,
17656 &mut swap.incumbent_rx,
17657 101,
17658 "aft",
17659 "swap-caller",
17660 )
17661 .await;
17662 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
17663
17664 register_candidate(&swap, None);
17665 cutover(&swap);
17666
17667 let ack = handler
17669 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
17670 .await;
17671 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
17672 if module_loop_error.is_some() {
17673 handler.cleanup_connection(INCUMBENT).unwrap();
17676 }
17677
17678 assert!(
17680 cotenant_rx.try_recv().is_err(),
17681 "the co-tenant route on the incumbent was torn down by one late ack: \
17682 {module_loop_error:?}"
17683 );
17684 assert!(matches!(
17685 swap.forwarding
17686 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
17687 .unwrap(),
17688 DataRoute::Client(DataRouteState::Bound(_))
17689 ));
17690 assert_eq!(module_loop_error, None);
17691 assert!(swap
17692 .registry
17693 .get_module_by_connection(INCUMBENT)
17694 .unwrap()
17695 .is_some());
17696
17697 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
17699 .await
17700 .expect("the incumbent is told to drop the abandoned binding")
17701 .unwrap()
17702 .frame;
17703 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17704 assert_eq!(goodbye.header.channel, abandoned_channel);
17705 assert_eq!(goodbye.header.epoch, abandoned_epoch);
17706 assert!(swap.incumbent_rx.try_recv().is_err());
17707
17708 let response = caller_task.await.unwrap();
17710 assert_eq!(response.len(), 1);
17711 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17712 assert!(caller_rx.try_recv().is_err());
17713
17714 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
17718 let counters = handler.counters().snapshot();
17719 assert_eq!(
17720 keyed_total(&counters, "route_open_accepted_by_principal"),
17721 1
17722 );
17723 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
17724 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
17725 }
17726
17727 #[tokio::test]
17731 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
17732 let mut swap = swap_with_incumbent().await;
17733 register_candidate(&swap, None);
17734 let incumbent = cutover(&swap);
17735 swap.forwarding
17736 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
17737 .unwrap()
17738 .expect("the incumbent is still registered");
17739
17740 let client = ConnectionId::new(33);
17741 let (client_ctx, mut client_rx) = route_ctx(client);
17742 let route_handler = swap.handler.clone();
17743 let open_ctx = RouteCtx {
17744 connection_id: client,
17745 egress: client_ctx.egress.clone(),
17746 };
17747 let mut route_task = tokio::spawn(async move {
17748 route_handler
17749 .handle_control_frame(
17750 &open_ctx,
17751 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
17752 )
17753 .await
17754 .unwrap()
17755 });
17756 let bind = tokio::select! {
17757 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
17758 response = &mut route_task => {
17759 let response = response.unwrap();
17760 panic!(
17761 "post-cutover route.open was refused instead of relayed to the candidate: {}",
17762 parse_error(&response[0])["code"]
17763 );
17764 }
17765 };
17766 swap.handler
17767 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
17768 .await
17769 .unwrap();
17770 assert!(route_task.await.unwrap().is_empty());
17771 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
17772 match swap
17773 .forwarding
17774 .lookup_data_route(client, channel, epoch)
17775 .unwrap()
17776 {
17777 DataRoute::Client(DataRouteState::Bound(route)) => {
17778 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
17779 }
17780 other => panic!("expected a bound route on the candidate, got {other:?}"),
17781 }
17782 assert!(swap.incumbent_rx.try_recv().is_err());
17783 }
17784
17785 #[tokio::test]
17790 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
17791 let swap = swap_with_incumbent().await;
17792 register_candidate(&swap, Some(false));
17793 let update = Frame::build(
17794 FrameType::Request,
17795 control_flags(),
17796 0,
17797 0,
17798 55,
17799 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
17800 provides: manifest("aft", PROTOCOL_VERSION).provides,
17801 capabilities: None,
17802 ready: Some(true),
17803 })
17804 .unwrap(),
17805 )
17806 .unwrap();
17807
17808 let replies = swap
17809 .handler
17810 .handle_control_frame(&swap.candidate_ctx, update)
17811 .await
17812 .unwrap();
17813
17814 assert_eq!(replies.len(), 1);
17815 assert_eq!(
17816 replies[0].header.ty,
17817 FrameType::Response,
17818 "candidate catalog.update was refused: {:?}",
17819 serde_json::from_slice::<Value>(&replies[0].body).ok()
17820 );
17821 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
17822 assert_eq!(
17823 swap.registry
17824 .get_module("aft")
17825 .unwrap()
17826 .unwrap()
17827 .connection_id,
17828 INCUMBENT
17829 );
17830 }
17831 }
17832
17833 mod swap_admission {
17837 use super::*;
17838
17839 const INCUMBENT_NONCE: &str = "incumbent-nonce";
17840 const CANDIDATE_NONCE: &str = "candidate-nonce";
17841
17842 fn handler_with_incumbent(
17843 module_id: &str,
17844 reserved: bool,
17845 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
17846 let registry = Arc::new(Registry::default());
17847 let supervisor = SupervisorHandle::new();
17848 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
17849 if reserved {
17850 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
17851 }
17852 let handler =
17853 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
17854 let incumbent = handler
17855 .handle_control(
17856 ConnectionId::new(1),
17857 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
17858 )
17859 .unwrap();
17860 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
17861 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
17862 (registry, supervisor, handler)
17863 }
17864
17865 #[test]
17871 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
17872 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
17873
17874 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
17875 let replies = handler
17876 .handle_control(
17877 ConnectionId::new(connection),
17878 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
17879 )
17880 .unwrap();
17881 assert_eq!(replies[0].header.ty, FrameType::Error);
17882 assert_eq!(
17883 parse_error(&replies[0])["code"],
17884 "swap_token_invalid",
17885 "nonce {nonce:?}"
17886 );
17887 }
17888 assert!(registry.get_candidate("aft").unwrap().is_none());
17889 assert_eq!(
17890 registry.get_module("aft").unwrap().unwrap().connection_id,
17891 ConnectionId::new(1)
17892 );
17893
17894 let admitted = handler
17897 .handle_control(
17898 ConnectionId::new(4),
17899 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
17900 )
17901 .unwrap();
17902 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
17903 assert_eq!(
17904 registry
17905 .get_candidate("aft")
17906 .unwrap()
17907 .unwrap()
17908 .connection_id,
17909 ConnectionId::new(4)
17910 );
17911 assert_eq!(
17912 registry.get_module("aft").unwrap().unwrap().connection_id,
17913 ConnectionId::new(1),
17914 "the candidate must not take the active slot"
17915 );
17916 let replayed = handler
17917 .handle_control(
17918 ConnectionId::new(5),
17919 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
17920 )
17921 .unwrap();
17922 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
17923
17924 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
17928 let squatter = handler
17929 .handle_control(
17930 ConnectionId::new(6),
17931 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
17932 )
17933 .unwrap();
17934 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
17935 assert!(
17936 registry.get_module("aft").unwrap().is_none(),
17937 "a squatter took the active slot of an id being swapped"
17938 );
17939 }
17940
17941 #[test]
17946 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
17947 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
17948
17949 let replies = handler
17950 .handle_control(
17951 ConnectionId::new(2),
17952 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17953 )
17954 .unwrap();
17955
17956 assert_eq!(
17957 replies[0].header.ty,
17958 FrameType::HelloAck,
17959 "reserved candidate refused: {:?}",
17960 serde_json::from_slice::<Value>(&replies[0].body).ok()
17961 );
17962 assert_eq!(
17963 registry
17964 .get_candidate("vault")
17965 .unwrap()
17966 .unwrap()
17967 .connection_id,
17968 ConnectionId::new(2)
17969 );
17970 }
17971
17972 #[test]
17975 fn without_an_open_swap_the_ordinary_gates_decide() {
17976 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
17977 supervisor.close_swap("vault");
17978
17979 let candidate = handler
17980 .handle_control(
17981 ConnectionId::new(2),
17982 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17983 )
17984 .unwrap();
17985 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
17986 let duplicate = handler
17987 .handle_control(
17988 ConnectionId::new(3),
17989 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
17990 )
17991 .unwrap();
17992 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
17993 assert!(registry.get_candidate("vault").unwrap().is_none());
17994 }
17995 }
17996
17997 mod scopes {
18001 mod idle_tests {
18002 use super::*;
18003 include!("scope_idle_tests.rs");
18004 }
18005 mod route_open_timings {
18006 use super::*;
18007 include!("route_open_timing_tests.rs");
18008 }
18009 mod route_open_refusal_log {
18010 use super::*;
18011 include!("route_open_refusal_log_tests.rs");
18012 }
18013 mod activity {
18014 include!("activity_scope_tests.rs");
18015 }
18016 mod delta {
18017 include!("scope_handler_delta_tests.rs");
18018 }
18019 mod call_modes {
18020 include!("scope_call_mode_tests.rs");
18021 }
18022 mod federated {
18023 include!("scope_federated_tests.rs");
18024 }
18025 mod authorize {
18026 include!("route_authorize_tests.rs");
18027 }
18028 #[tokio::test]
18029 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
18030 let rig = rig().await;
18031 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
18032 "s".into(),
18033 1,
18034 "tool".into(),
18035 ))
18036 .unwrap();
18037 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
18038 let (direct, _rx) = wide_ctx(999);
18039 for (ctx, expected) in [
18040 (&rig.owner, FrameType::Response),
18041 (&direct, FrameType::Error),
18042 ] {
18043 let frame =
18044 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
18045 .unwrap();
18046 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
18047 assert_eq!(replies[0].header.ty, expected);
18048 if expected == FrameType::Response {
18049 let reply: subc_control::ScopeServedByReply =
18050 serde_json::from_slice(&replies[0].body).unwrap();
18051 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
18052 } else {
18053 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
18054 }
18055 }
18056 let reply = call(
18057 &rig.handler,
18058 &rig.owner,
18059 &ModuleControlRequestFromModule::ScopeDescribe {
18060 owner: Principal::Reserved {
18061 module_id: OWNER.into(),
18062 },
18063 scope_ref: "s".into(),
18064 },
18065 )
18066 .await;
18067 assert!(matches!(
18068 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
18069 ModuleControlResponseToModule::ScopeDescribe { .. }
18070 ));
18071 }
18072
18073 use subc_protocol::scope::{
18074 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
18075 ScopeStatus,
18076 };
18077
18078 use super::*;
18079
18080 const OWNER: &str = "prefrontal-core";
18081
18082 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
18083 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
18084 }
18085
18086 async fn call(
18087 handler: &ControlHandler,
18088 ctx: &RouteCtx,
18089 request: &ModuleControlRequestFromModule,
18090 ) -> Frame {
18091 let body = serde_json::to_vec(request).unwrap();
18092 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
18093 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
18094 assert_eq!(replies.len(), 1, "{replies:?}");
18095 replies.pop().unwrap()
18096 }
18097
18098 async fn sync(
18099 handler: &ControlHandler,
18100 ctx: &RouteCtx,
18101 generation: u64,
18102 scopes: Vec<ScopeRecord>,
18103 ) -> Result<ModuleControlResponseToModule, String> {
18104 let reply = call(
18105 handler,
18106 ctx,
18107 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
18108 )
18109 .await;
18110 match reply.header.ty {
18111 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
18112 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
18113 }
18114 }
18115
18116 async fn describe(
18117 handler: &ControlHandler,
18118 ctx: &RouteCtx,
18119 owner: &str,
18120 scope_ref: &str,
18121 ) -> ModuleControlResponseToModule {
18122 let reply = call(
18123 handler,
18124 ctx,
18125 &ModuleControlRequestFromModule::ScopeDescribe {
18126 owner: Principal::Reserved {
18127 module_id: owner.to_string(),
18128 },
18129 scope_ref: scope_ref.to_string(),
18130 },
18131 )
18132 .await;
18133 assert_eq!(
18134 reply.header.ty,
18135 FrameType::Response,
18136 "{:?}",
18137 parse_error(&reply)
18138 );
18139 serde_json::from_slice(&reply.body).unwrap()
18140 }
18141
18142 #[tokio::test]
18143 async fn scope_describe_never_echoes_bind_proof_or_hash() {
18144 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
18145 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18146 let mut rig = rig().await;
18147 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18148 .await;
18149 let _bound = rig
18150 .bound(
18151 Some(AFT),
18152 PLEXUS,
18153 Some(
18154 rig.selector("s", Some(1))
18155 .with_bind_proof(Some(PROOF.into())),
18156 ),
18157 )
18158 .await;
18159 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
18160 let output = serde_json::to_string(&reply).unwrap();
18161 assert!(output.contains("\"status\":\"live\""));
18162 assert!(!output.contains(PROOF));
18163 assert!(!output.contains(HASH));
18164 }
18165
18166 async fn module(
18168 handler: &ControlHandler,
18169 connection: u64,
18170 module_id: &str,
18171 nonce: Option<&str>,
18172 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
18173 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
18174 hello_via_sink(
18175 handler,
18176 &ctx,
18177 &mut rx,
18178 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
18179 )
18180 .await;
18181 (ctx, rx)
18182 }
18183
18184 #[tokio::test]
18187 async fn a_client_connection_cannot_sync_or_describe() {
18188 let handler = ControlHandler::new(Arc::new(Registry::default()));
18189 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
18190 for request in [
18191 ModuleControlRequestFromModule::ScopeSync {
18192 generation: 1,
18193 scopes: vec![head("s", 1)],
18194 },
18195 ModuleControlRequestFromModule::ScopeDescribe {
18196 owner: Principal::Direct,
18197 scope_ref: "s".to_string(),
18198 },
18199 ] {
18200 let reply = call(&handler, &ctx, &request).await;
18201 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
18202 }
18203 assert!(
18204 !handler
18205 .scopes
18206 .read()
18207 .unwrap()
18208 .describe(
18209 &Principal::Reserved {
18210 module_id: OWNER.to_string()
18211 },
18212 "s"
18213 )
18214 .owner_synced
18215 );
18216 }
18217
18218 #[tokio::test]
18221 async fn a_module_without_a_supervised_launch_cannot_sync() {
18222 let handler = ControlHandler::new(Arc::new(Registry::default()));
18223 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
18224 assert_eq!(
18225 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
18226 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18227 );
18228 }
18229
18230 #[tokio::test]
18231 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
18232 let supervisor = SupervisorHandle::new();
18233 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
18234 let handler = ControlHandler::new(Arc::new(Registry::default()))
18235 .with_supervisor(supervisor.clone());
18236 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
18237 sync(&handler, &incumbent, 1, vec![head("s", 1)])
18238 .await
18239 .expect("the current launch syncs");
18240
18241 supervisor.open_swap(OWNER, "n2".to_string());
18244 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
18245 assert_eq!(
18246 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
18247 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18248 );
18249 sync(&handler, &incumbent, 2, vec![head("s", 1)])
18250 .await
18251 .expect("the serving owner syncs during the swap");
18252
18253 supervisor.close_swap(OWNER);
18256 assert_eq!(
18257 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
18258 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18259 );
18260 sync(&handler, &incumbent, 3, vec![head("s", 1)])
18261 .await
18262 .expect("the serving owner syncs after the rollback");
18263 handler.cleanup_connection(candidate.connection_id).unwrap();
18264
18265 supervisor.open_swap(OWNER, "n3".to_string());
18270 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
18271 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
18272 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
18273 .await
18274 .expect("the promoted launch takes authority");
18275 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
18276 panic!("unexpected reply {reply:?}");
18277 };
18278 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
18279 assert_eq!(
18280 sync(&handler, &incumbent, 4, Vec::new()).await,
18281 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18282 );
18283 }
18284
18285 #[tokio::test]
18288 async fn closing_the_authority_connection_frees_sync_authority() {
18289 let supervisor = SupervisorHandle::new();
18290 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
18291 let handler = ControlHandler::new(Arc::new(Registry::default()))
18292 .with_supervisor(supervisor.clone());
18293 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
18294 sync(&handler, &first, 10, vec![head("s", 1)])
18295 .await
18296 .unwrap();
18297 handler.cleanup_connection(first.connection_id).unwrap();
18298
18299 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
18300 sync(&handler, &second, 1, vec![head("s", 1)])
18301 .await
18302 .expect("the next connection takes the released authority");
18303 }
18304
18305 #[tokio::test]
18306 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
18307 let supervisor = SupervisorHandle::new();
18308 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
18309 let handler =
18310 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
18311 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
18312 sync(&handler, &first, 10, vec![head("s", 1)])
18313 .await
18314 .unwrap();
18315 handler
18316 .handle_control_frame(
18317 &first,
18318 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
18319 )
18320 .await
18321 .unwrap();
18322 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
18323 sync(&handler, &second, 1, vec![head("s", 1)])
18324 .await
18325 .expect("GOODBYE releases authority even if the old socket remains open");
18326 }
18327
18328 #[tokio::test]
18329 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
18330 let registry = Arc::new(Registry::default());
18331 let supervisor_handle = SupervisorHandle::new();
18332 let supervisor =
18333 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
18334 .with_handle(supervisor_handle.clone())
18335 .with_daemon_incarnation("incarnation-7".to_string());
18336 supervisor
18339 .supervise_configured(
18340 ModuleSpec {
18341 carry_containment: false,
18342 module_id: OWNER.to_string(),
18343 program: PathBuf::from("/nonexistent/prefrontal-core"),
18344 args: Vec::new(),
18345 env: Vec::new(),
18346 reserved: false,
18347 reserved_prefixes: Vec::new(),
18348 protocol: ModuleProtocol::Subc,
18349 overlap: Default::default(),
18350 },
18351 false,
18352 )
18353 .unwrap();
18354 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
18355 let handler =
18356 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
18357 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
18358
18359 let ModuleControlResponseToModule::ScopeDescribe {
18361 status,
18362 daemon_incarnation,
18363 owner_synced,
18364 owner_configured,
18365 scope,
18366 ..
18367 } = describe(&handler, &reader, OWNER, "s").await
18368 else {
18369 panic!("not a describe reply");
18370 };
18371 assert_eq!(status, ScopeStatus::NotLive);
18372 assert_eq!(daemon_incarnation, "incarnation-7");
18373 assert!(!owner_synced);
18374 assert!(owner_configured);
18375 assert!(scope.is_none());
18376
18377 let ModuleControlResponseToModule::ScopeDescribe {
18380 status,
18381 owner_configured,
18382 ..
18383 } = describe(&handler, &reader, "ghost", "s").await
18384 else {
18385 panic!("not a describe reply");
18386 };
18387 assert_eq!(status, ScopeStatus::NotLive);
18388 assert!(!owner_configured);
18389
18390 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
18392 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
18393 let ModuleControlResponseToModule::ScopeDescribe {
18394 status,
18395 scope_epoch,
18396 owner_synced,
18397 scope,
18398 ..
18399 } = describe(&handler, &reader, OWNER, "s").await
18400 else {
18401 panic!("not a describe reply");
18402 };
18403 assert_eq!(status, ScopeStatus::Live);
18404 assert_eq!(scope_epoch, Some(4));
18405 assert!(owner_synced);
18406 let stamp = scope.expect("a live scope carries its stamp");
18407 assert!(
18408 stamp.owner_authorized,
18409 "prefrontal-core is the default authority"
18410 );
18411 assert_eq!(stamp.kind, ScopeKind::Head);
18412 }
18413
18414 #[tokio::test]
18415 async fn scope_authority_owners_decides_owner_authorized() {
18416 let supervisor = SupervisorHandle::new();
18417 supervisor.set_spawn_nonce("broca", "b1".to_string());
18418 let handler = ControlHandler::new(Arc::new(Registry::default()))
18419 .with_supervisor(supervisor)
18420 .with_scope_authority_owners(vec!["broca".to_string()]);
18421 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
18422 let mut gated = head("s", 1);
18423 gated.attributes.agent_id = Some("agent".to_string());
18424 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
18425 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
18426 describe(&handler, &broca, "broca", "s").await
18427 else {
18428 panic!("not a describe reply");
18429 };
18430 assert!(scope.unwrap().owner_authorized);
18431 }
18432
18433 #[tokio::test]
18437 async fn scope_ops_and_the_scopes_capability_are_advertised() {
18438 let handler = ControlHandler::new(Arc::new(Registry::default()));
18439 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
18440 let ack = hello_via_sink(
18441 &handler,
18442 &ctx,
18443 &mut rx,
18444 hello_frame("m", PROTOCOL_VERSION, 1),
18445 )
18446 .await;
18447 let ack = parse_ack(&ack);
18448 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
18449 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
18450 }
18451 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
18452
18453 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
18454 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
18455 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
18456 let reply = handler
18457 .handle_control_frame(&client, frame)
18458 .await
18459 .unwrap()
18460 .pop()
18461 .unwrap();
18462 let ClientControlResponse::ServerDescribe { capabilities, .. } =
18463 serde_json::from_slice(&reply.body).unwrap()
18464 else {
18465 panic!("not a server.describe reply");
18466 };
18467 assert!(
18468 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
18469 "{capabilities:?}"
18470 );
18471 }
18472
18473 const PLEXUS: &str = "plexus";
18476 const OTHER: &str = "other";
18477 const AFT: &str = "aft";
18478 const BROCA: &str = "broca";
18479 const MAGIC: &str = "magic-context";
18480
18481 fn nonce(module_id: &str) -> String {
18482 format!("nonce-{module_id}")
18483 }
18484
18485 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
18486 let (tx, rx) = mpsc::channel(64);
18487 (
18488 RouteCtx {
18489 connection_id: ConnectionId::new(connection),
18490 egress: FrameSink::new(tx),
18491 },
18492 rx,
18493 )
18494 }
18495
18496 struct Rig {
18500 handler: ControlHandler,
18501 forwarding: Arc<ForwardingTable>,
18502 owner: RouteCtx,
18503 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18504 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
18505 generation: u64,
18506 next_connection: u64,
18507 _supervisor: Supervisor,
18508 }
18509
18510 async fn rig() -> Rig {
18511 rig_with_flow_support(true).await
18512 }
18513
18514 async fn rig_with_flow_support(flow_support: bool) -> Rig {
18515 rig_with_scope_capabilities(flow_support, false).await
18516 }
18517
18518 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
18519 let mut provides = Vec::new();
18520 if flow_support {
18521 provides.push("flow-scopes/v1");
18522 }
18523 if run_support {
18524 provides.push("agent-run-scopes/v1");
18525 }
18526 rig_with_target_capabilities(&provides).await
18527 }
18528
18529 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
18530 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
18531 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18532 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18535 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
18536 Frame::build(
18537 FrameType::Hello,
18538 control_flags(),
18539 0,
18540 0,
18541 connection,
18542 serde_json::to_vec(&body).unwrap(),
18543 )
18544 .unwrap()
18545 }
18546
18547 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
18548 let registry = Arc::new(Registry::default());
18549 let forwarding = Arc::new(ForwardingTable::default());
18550 let supervisor_handle = SupervisorHandle::new();
18551 let supervisor =
18552 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
18553 .with_handle(supervisor_handle.clone());
18554 supervisor
18555 .supervise_configured(
18556 ModuleSpec {
18557 carry_containment: false,
18558 module_id: OWNER.to_string(),
18559 program: PathBuf::from("/nonexistent/prefrontal-core"),
18560 args: Vec::new(),
18561 env: Vec::new(),
18562 reserved: false,
18563 reserved_prefixes: Vec::new(),
18564 protocol: ModuleProtocol::Subc,
18565 overlap: Default::default(),
18566 },
18567 false,
18568 )
18569 .unwrap();
18570 for module_id in [OWNER, AFT, BROCA, MAGIC] {
18571 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
18572 }
18573 let handler =
18574 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18575 .with_supervisor(supervisor_handle);
18576 let (owner, mut owner_rx) = wide_ctx(1);
18577 hello_via_sink(
18578 &handler,
18579 &owner,
18580 &mut owner_rx,
18581 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
18582 )
18583 .await;
18584 let mut modules = BTreeMap::new();
18585 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
18586 let (ctx, mut rx) = wide_ctx(connection);
18587 let hello = hello_with_capabilities(module_id, connection, provides);
18588 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
18589 modules.insert(module_id.to_string(), (ctx, rx));
18590 }
18591 Rig {
18592 handler,
18593 forwarding,
18594 owner,
18595 _owner_rx: owner_rx,
18596 modules,
18597 generation: 0,
18598 next_connection: 100,
18599 _supervisor: supervisor,
18600 }
18601 }
18602
18603 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
18604 ScopeCarrier::new(Principal::Reserved {
18605 module_id: module_id.to_string(),
18606 })
18607 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
18608 }
18609
18610 fn session(scope_epoch: u64) -> ScopeRecord {
18613 let mut record = head("s", scope_epoch);
18614 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
18615 record.attributes.agent_id = Some("agent-1".to_string());
18616 record.attributes.delegates = true;
18617 record
18618 }
18619
18620 impl Rig {
18621 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
18622 self.generation += 1;
18623 sync(&self.handler, &self.owner, self.generation, scopes)
18624 .await
18625 .expect("the owner's sync is accepted");
18626 }
18627
18628 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18629 ScopeSelector::new(
18630 Principal::Reserved {
18631 module_id: OWNER.to_string(),
18632 },
18633 scope_ref.to_string(),
18634 scope_epoch,
18635 )
18636 }
18637
18638 fn open_frame(
18639 &mut self,
18640 opener: Option<&str>,
18641 target: &str,
18642 scope: Option<ScopeSelector>,
18643 ) -> (
18644 RouteCtx,
18645 mpsc::Receiver<crate::router::OutboundFrame>,
18646 Frame,
18647 ) {
18648 self.next_connection += 1;
18649 let (ctx, rx) = wide_ctx(self.next_connection);
18650 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
18651 busy_refusals: None,
18652 target: RouteTarget::ToolProvider {
18653 module_id: target.to_string(),
18654 },
18655 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
18656 consumer_identity: opener.map(|module_id| ConsumerIdentity {
18657 module_id: module_id.to_string(),
18658 launch_nonce: nonce(module_id),
18659 }),
18660 consumer_capabilities: None,
18661 role_versions: None,
18662 admission_facts: None,
18663 scope,
18664 })
18665 .unwrap();
18666 let frame = Frame::build(
18667 FrameType::Request,
18668 control_flags(),
18669 0,
18670 0,
18671 self.next_connection,
18672 body,
18673 )
18674 .unwrap();
18675 (ctx, rx, frame)
18676 }
18677
18678 async fn refused(
18680 &mut self,
18681 opener: Option<&str>,
18682 target: &str,
18683 scope: Option<ScopeSelector>,
18684 ) -> String {
18685 self.refusal_body(opener, target, scope).await["code"]
18686 .as_str()
18687 .unwrap()
18688 .to_string()
18689 }
18690
18691 async fn refusal_body(
18692 &mut self,
18693 opener: Option<&str>,
18694 target: &str,
18695 scope: Option<ScopeSelector>,
18696 ) -> Value {
18697 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
18698 let replies = tokio::time::timeout(
18699 Duration::from_secs(2),
18700 self.handler.handle_control_frame(&ctx, frame),
18701 )
18702 .await
18703 .expect("the open must be refused before waiting for a bind ack")
18704 .unwrap();
18705 assert_eq!(replies.len(), 1, "{replies:?}");
18706 assert_eq!(replies[0].header.ty, FrameType::Error);
18707 let (_, module_rx) = self.modules.get_mut(target).unwrap();
18708 assert!(
18709 module_rx.try_recv().is_err(),
18710 "a refused open relays nothing"
18711 );
18712 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
18713 parse_error(&replies[0])
18714 }
18715
18716 async fn relayed(
18718 &mut self,
18719 opener: Option<&str>,
18720 target: &str,
18721 scope: Option<ScopeSelector>,
18722 ) -> Relayed {
18723 self.relayed_with_legacy_root(opener, target, scope, None)
18724 .await
18725 }
18726
18727 async fn relayed_with_legacy_root(
18728 &mut self,
18729 opener: Option<&str>,
18730 target: &str,
18731 scope: Option<ScopeSelector>,
18732 legacy_root: Option<&std::path::Path>,
18733 ) -> Relayed {
18734 let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
18735 if let Some(root) = legacy_root {
18736 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
18737 body["identity"]["project_root"] = serde_json::json!(root);
18738 frame.body = serde_json::to_vec(&body).unwrap();
18739 }
18740 let handler = self.handler.clone();
18741 let task_ctx = ctx.clone();
18742 let task = tokio::spawn(async move {
18743 handler
18744 .handle_control_frame(&task_ctx, frame)
18745 .await
18746 .unwrap()
18747 });
18748 let (_, module_rx) = self.modules.get_mut(target).unwrap();
18749 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
18750 .await
18751 .expect("the target receives the relayed route.bind")
18752 .unwrap()
18753 .frame;
18754 Relayed {
18755 target: target.to_string(),
18756 client: ctx,
18757 client_rx: rx,
18758 task,
18759 bind,
18760 }
18761 }
18762
18763 async fn ack(&self, relayed: &Relayed) {
18764 let (module, _) = &self.modules[&relayed.target];
18765 self.handler
18766 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
18767 .await
18768 .unwrap();
18769 }
18770
18771 async fn bound(
18773 &mut self,
18774 opener: Option<&str>,
18775 target: &str,
18776 scope: Option<ScopeSelector>,
18777 ) -> Bound {
18778 let relayed = self.relayed(opener, target, scope).await;
18779 self.commit_relayed(relayed).await
18780 }
18781
18782 async fn bound_with_legacy_root(
18783 &mut self,
18784 opener: Option<&str>,
18785 target: &str,
18786 scope: Option<ScopeSelector>,
18787 root: &std::path::Path,
18788 ) -> Bound {
18789 let relayed = self
18790 .relayed_with_legacy_root(opener, target, scope, Some(root))
18791 .await;
18792 self.commit_relayed(relayed).await
18793 }
18794
18795 async fn commit_relayed(&self, relayed: Relayed) -> Bound {
18796 self.ack(&relayed).await;
18797 let Relayed {
18798 target,
18799 client,
18800 mut client_rx,
18801 task,
18802 bind,
18803 } = relayed;
18804 assert!(
18805 task.await.unwrap().is_empty(),
18806 "the open is answered by commit"
18807 );
18808 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
18809 Bound {
18810 target,
18811 client,
18812 client_rx,
18813 channel,
18814 epoch,
18815 bind,
18816 }
18817 }
18818
18819 fn live(&self, route: &Bound) -> bool {
18820 matches!(
18821 self.forwarding
18822 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18823 .unwrap(),
18824 DataRoute::Client(DataRouteState::Bound(_))
18825 )
18826 }
18827 }
18828
18829 struct Relayed {
18830 target: String,
18831 client: RouteCtx,
18832 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18833 task: tokio::task::JoinHandle<Vec<Frame>>,
18834 bind: Frame,
18835 }
18836
18837 struct Bound {
18838 target: String,
18839 client: RouteCtx,
18840 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18841 channel: u16,
18842 epoch: u32,
18843 bind: Frame,
18844 }
18845
18846 impl Bound {
18847 fn closed_reason(&mut self) -> RouteCloseReason {
18850 let mut reason = None;
18851 let mut goodbye = false;
18852 while let Ok(outbound) = self.client_rx.try_recv() {
18853 let frame = outbound.frame;
18854 match frame.header.ty {
18855 FrameType::Goodbye => {
18856 assert_eq!(
18857 (frame.header.channel, frame.header.epoch),
18858 (self.channel, self.epoch)
18859 );
18860 goodbye = true;
18861 }
18862 FrameType::Push => {
18863 let ClientControlPush::RouteClosed {
18864 reason: r,
18865 module_id,
18866 ..
18867 } = serde_json::from_slice(&frame.body).unwrap()
18868 else {
18869 panic!("unexpected push");
18870 };
18871 assert_eq!(module_id, self.target);
18872 reason = Some(r);
18873 }
18874 other => panic!("unexpected frame {other:?}"),
18875 }
18876 }
18877 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
18878 reason.expect("the client is told why the route closed")
18879 }
18880
18881 fn untouched(&mut self) -> bool {
18882 self.client_rx.try_recv().is_err()
18883 }
18884
18885 fn stamp(&self) -> Option<ScopeStamp> {
18886 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
18887 ModuleControlRequest::RouteBind { scope, .. } => scope,
18888 other => panic!("expected a route.bind, got {other:?}"),
18889 }
18890 }
18891 }
18892
18893 #[tokio::test]
18894 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
18895 ) {
18896 let mut rig = rig().await;
18897 let mut record = session(1);
18898 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
18899 record.child_owners = vec![Principal::Reserved {
18900 module_id: MAGIC.to_string(),
18901 }];
18902 rig.sync(vec![record]).await;
18903 let scope = || Some(rig_selector("s", Some(1)));
18904
18905 rig.bound(Some(OWNER), PLEXUS, scope()).await;
18908 rig.bound(Some(AFT), OTHER, scope()).await;
18909 rig.bound(Some(BROCA), PLEXUS, scope()).await;
18910
18911 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
18915 assert_eq!(
18916 rig.refused(opener, target, scope()).await,
18917 error_codes::SCOPE_NOT_CARRIER,
18918 "{opener:?} -> {target}"
18919 );
18920 }
18921 }
18922
18923 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18924 ScopeSelector::new(
18925 Principal::Reserved {
18926 module_id: OWNER.to_string(),
18927 },
18928 scope_ref.to_string(),
18929 scope_epoch,
18930 )
18931 }
18932
18933 #[tokio::test]
18934 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
18935 let mut rig = rig().await;
18936 rig.sync(vec![session(1)]).await;
18937 for opener in [OWNER, AFT] {
18938 assert_eq!(
18939 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
18940 .await,
18941 error_codes::SCOPE_EPOCH_REQUIRED,
18942 "{opener}"
18943 );
18944 }
18945 }
18946
18947 #[tokio::test]
18948 async fn admission_separates_not_synced_not_live_and_ended() {
18949 let mut rig = rig().await;
18950 let code = rig
18952 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18953 .await;
18954 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
18955 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
18956
18957 let ghost = ScopeSelector::new(
18959 Principal::Reserved {
18960 module_id: "ghost".to_string(),
18961 },
18962 "s".to_string(),
18963 Some(1),
18964 );
18965 assert_eq!(
18966 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
18967 error_codes::SCOPE_NOT_LIVE
18968 );
18969
18970 rig.sync(vec![session(2)]).await;
18971 assert_eq!(
18972 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
18973 .await,
18974 error_codes::SCOPE_NOT_LIVE
18975 );
18976 for epoch in [1, 3] {
18977 assert_eq!(
18978 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
18979 .await,
18980 error_codes::SCOPE_ENDED,
18981 "epoch {epoch}"
18982 );
18983 }
18984 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
18986 .await;
18987 }
18988
18989 #[tokio::test]
18990 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
18991 let mut rig = rig().await;
18992 rig.sync(vec![session(1)]).await;
18993 let route = rig
18994 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18995 .await;
18996 let stamp = route.stamp().expect("a scoped bind carries the stamp");
18997 assert_eq!(stamp.scope_ref, "s");
18998 assert_eq!(stamp.scope_epoch, 1);
18999 assert_eq!(stamp.kind, ScopeKind::Head);
19000 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
19001 assert!(stamp.attributes.delegates);
19002 assert!(stamp.owner_authorized);
19003 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
19004 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
19005
19006 let (broca, mut broca_rx) = wide_ctx(50);
19009 hello_via_sink(
19010 &rig.handler,
19011 &broca,
19012 &mut broca_rx,
19013 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
19014 )
19015 .await;
19016 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
19017 .await
19018 .unwrap();
19019 let own = ScopeSelector::new(
19020 Principal::Reserved {
19021 module_id: BROCA.to_string(),
19022 },
19023 "b".to_string(),
19024 Some(1),
19025 );
19026 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
19027 assert!(!route.stamp().unwrap().owner_authorized);
19028 }
19029
19030 #[tokio::test]
19031 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
19032 let mut rig = rig().await;
19033 let mut record = head("s", 1);
19034 record.carriers = vec![carrier(AFT, None)];
19035 let flow_id = "Flow:run-7/step_2!~";
19036 record.attributes.flow_id = Some(flow_id.to_string());
19037 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
19038 .await
19039 .unwrap();
19040 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
19041 panic!("not a sync reply");
19042 };
19043 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
19044 let route = rig
19045 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19046 .await;
19047 assert!(rig.live(&route), "the stamped bind committed");
19048 let stamp = route.stamp().expect("a flow scope carries a stamp");
19049 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
19050 assert_eq!(stamp.attributes.agent_id, None);
19051 assert!(!stamp.attributes.delegates);
19052 assert!(stamp.owner_authorized);
19053 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
19054 assert_eq!(unscoped.stamp(), None);
19055 }
19056
19057 #[tokio::test]
19058 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
19059 let mut rig = rig_with_flow_support(false).await;
19060 let mut record = session(1);
19061 record.attributes.flow_id = Some("flow:7".to_string());
19062 rig.sync(vec![record]).await;
19063 for opener in [OWNER, AFT] {
19064 let body = rig
19065 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
19066 .await;
19067 assert_eq!(body["code"], "target_flow_unsupported");
19068 let message = body["message"].as_str().unwrap();
19069 for required in [PLEXUS, "flow-scopes/v1"] {
19070 assert!(message.contains(required), "{message}");
19071 }
19072 }
19073 }
19074
19075 #[tokio::test]
19076 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
19077 let mut rig = rig_with_flow_support(true).await;
19078 let mut record = session(1);
19079 record.attributes.flow_id = Some("flow:7".to_string());
19080 rig.sync(vec![record]).await;
19081 for opener in [OWNER, AFT] {
19082 let route = rig
19083 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
19084 .await;
19085 assert!(rig.live(&route));
19086 assert_eq!(
19087 route.stamp().unwrap().attributes.flow_id.as_deref(),
19088 Some("flow:7")
19089 );
19090 }
19091 }
19092
19093 #[tokio::test]
19094 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
19095 use std::future::Future;
19096
19097 let mut rig = rig_with_flow_support(true).await;
19098 let mut record = session(1);
19099 record.attributes.flow_id = Some("flow:7".to_string());
19100 rig.sync(vec![record]).await;
19101 let (client, mut client_rx, frame) =
19102 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
19103 for _ in 0..64 {
19106 client.egress.try_send(route_bind_ack(1)).unwrap();
19107 }
19108 let handler = rig.handler.clone();
19109 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
19110 std::future::poll_fn(|cx| {
19111 assert!(open.as_mut().poll(cx).is_pending());
19112 std::task::Poll::Ready(())
19113 })
19114 .await;
19115 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19116
19117 let old_connection = rig.modules[PLEXUS].0.connection_id;
19118 rig.handler.cleanup_connection(old_connection).unwrap();
19119 let (replacement, mut replacement_rx) = wide_ctx(200);
19120 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
19121 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
19122 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
19123 let hello = Frame::build(
19124 FrameType::Hello,
19125 control_flags(),
19126 0,
19127 0,
19128 200,
19129 serde_json::to_vec(&body).unwrap(),
19130 )
19131 .unwrap();
19132 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
19133
19134 client_rx.try_recv().unwrap();
19135 let replies = tokio::time::timeout(Duration::from_secs(2), open)
19136 .await
19137 .expect("the replacement is refused without waiting for a bind ack")
19138 .unwrap();
19139 assert_eq!(replies.len(), 1);
19140 let body = parse_error(&replies[0]);
19141 assert_eq!(body["code"], "target_flow_unsupported");
19142 for required in [PLEXUS, "flow-scopes/v1"] {
19143 assert!(body["message"].as_str().unwrap().contains(required));
19144 }
19145 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
19146 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
19147 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19148 assert!(rig
19149 .handler
19150 .registry
19151 .get_module_by_connection(replacement.connection_id)
19152 .unwrap()
19153 .is_some());
19154 }
19155
19156 #[tokio::test]
19157 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
19158 use std::future::Future;
19159 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
19160 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
19161 const NEXT_HASH: &str =
19162 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
19163 let mut rig = rig().await;
19164 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
19165 record.carriers.push(ScopeCarrier::new(Principal::Direct));
19166 rig.sync(vec![record.clone()]).await;
19167 let (client, mut rx, frame) = rig.open_frame(
19168 None,
19169 PLEXUS,
19170 Some(
19171 rig.selector("s", Some(1))
19172 .with_bind_proof(Some(PROOF.into())),
19173 ),
19174 );
19175 for _ in 0..64 {
19178 client.egress.try_send(route_bind_ack(1)).unwrap();
19179 }
19180 let handler = rig.handler.clone();
19181 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
19182 std::future::poll_fn(|cx| {
19183 assert!(open.as_mut().poll(cx).is_pending());
19184 std::task::Poll::Ready(())
19185 })
19186 .await;
19187 record.bind_proof_sha256 = Some(NEXT_HASH.into());
19188 rig.sync(vec![record]).await;
19189 rx.try_recv().unwrap();
19190 let replies = tokio::time::timeout(Duration::from_secs(2), open)
19191 .await
19192 .unwrap()
19193 .unwrap();
19194 assert_eq!(
19195 parse_error(&replies[0])["code"],
19196 error_codes::SCOPE_BIND_PROOF_MISMATCH
19197 );
19198 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
19199 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
19200 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19201 }
19202
19203 #[tokio::test]
19204 async fn scope_bind_proof_reserved_openers_ignore_proof() {
19205 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
19206 let mut rig = rig().await;
19207 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
19208 .await;
19209 for opener in [OWNER, AFT] {
19210 for proof in [None, Some("malformed+proof=")] {
19211 let route = rig
19212 .bound(
19213 Some(opener),
19214 PLEXUS,
19215 Some(
19216 rig.selector("s", Some(1))
19217 .with_bind_proof(proof.map(str::to_string)),
19218 ),
19219 )
19220 .await;
19221 assert!(rig.live(&route));
19222 }
19223 }
19224 }
19225
19226 #[tokio::test]
19227 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
19228 use std::future::Future;
19229 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
19230 const NEXT_HASH: &str =
19231 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
19232 let mut rig = rig().await;
19233 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
19234 .await;
19235 let (client, mut rx, frame) =
19236 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
19237 for _ in 0..64 {
19238 client.egress.try_send(route_bind_ack(1)).unwrap();
19239 }
19240 let handler = rig.handler.clone();
19241 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
19242 std::future::poll_fn(|cx| {
19243 assert!(open.as_mut().poll(cx).is_pending());
19244 std::task::Poll::Ready(())
19245 })
19246 .await;
19247 rig.sync(vec![
19248 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
19249 ])
19250 .await;
19251 rx.try_recv().unwrap();
19252 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19253 let bind = std::future::poll_fn(|cx| {
19254 assert!(open.as_mut().poll(cx).is_pending());
19255 module_rx.poll_recv(cx)
19256 })
19257 .await
19258 .unwrap()
19259 .frame;
19260 handler
19261 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
19262 .await
19263 .unwrap();
19264 let replies = tokio::time::timeout(Duration::from_secs(2), open)
19265 .await
19266 .unwrap()
19267 .unwrap();
19268 assert_eq!(replies.len(), 1);
19269 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
19272 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19273 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
19274 let route = rig
19275 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
19276 .await;
19277 assert!(rig.live(&route));
19278 }
19279
19280 #[tokio::test]
19281 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
19282 {
19283 let mut rig = rig_with_flow_support(false).await;
19284 rig.sync(vec![session(1)]).await;
19285 let route = rig
19286 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19287 .await;
19288 assert!(rig.live(&route));
19289 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
19290 let mut record = session(1);
19291 record.attributes.flow_id = Some("flow:7".to_string());
19292 rig.sync(vec![record]).await;
19293 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
19294 assert!(rig.live(&unscoped));
19295 assert_eq!(unscoped.stamp(), None);
19296 }
19297
19298 #[tokio::test]
19299 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
19300 let mut rig = rig().await;
19301 let mut record = session(1);
19302 record.attributes.flow_id = Some("flow:7".to_string());
19303 rig.sync(vec![record.clone()]).await;
19304 let before = rig.forwarding.published_scope_tag(OWNER, "s");
19305 let mut owner_route = rig
19306 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
19307 .await;
19308 let mut carrier_route = rig
19309 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
19310 .await;
19311 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
19312 rig.sync(vec![record.clone()]).await;
19313 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
19314 assert!(rig.live(&owner_route) && owner_route.untouched());
19315 assert!(rig.live(&carrier_route) && carrier_route.untouched());
19316
19317 record.attributes.flow_id = Some("flow:8".to_string());
19318 rig.sync(vec![record]).await;
19319 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
19320 let before = before.unwrap();
19321 assert_eq!(after.scope_epoch, before.scope_epoch);
19322 assert!(after.version > before.version);
19323 for route in [&mut owner_route, &mut carrier_route] {
19324 assert!(!rig.live(route));
19325 assert_eq!(
19326 route.closed_reason(),
19327 RouteCloseReason::ScopeDelegationChanged
19328 );
19329 }
19330 assert!(rig.live(&unscoped) && unscoped.untouched());
19331 for target in [PLEXUS, OTHER] {
19334 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
19335 let goodbye = module_rx
19336 .try_recv()
19337 .expect("the provider sees the drain")
19338 .frame;
19339 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19340 assert!(module_rx.try_recv().is_err());
19341 }
19342 let rebound = rig
19343 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19344 .await;
19345 assert_eq!(
19346 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
19347 Some("flow:8")
19348 );
19349 }
19350
19351 #[tokio::test]
19356 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
19357 let mut rig = rig().await;
19358 rig.sync(vec![session(1)]).await;
19359 let mut cotenant = rig
19360 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
19361 .await;
19362
19363 let mut changed = session(1);
19364 changed.child_owners.push(Principal::Reserved {
19365 module_id: MAGIC.to_string(),
19366 });
19367 let mut ended = None;
19368 for (code, next) in [
19369 (error_codes::SCOPE_CHANGED, vec![changed]),
19370 (error_codes::SCOPE_ENDED, Vec::new()),
19371 ] {
19372 let relayed = rig
19373 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19374 .await;
19375 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
19376 ended = Some(next.is_empty());
19377 rig.sync(next).await;
19378 rig.ack(&relayed).await;
19379 let replies = relayed.task.await.unwrap();
19380 assert_eq!(replies.len(), 1, "{replies:?}");
19381 assert_eq!(parse_error(&replies[0])["code"], code);
19382 assert_eq!(
19383 subc_protocol::error_codes::is_retryable_route_open(code),
19384 code == error_codes::SCOPE_CHANGED
19385 );
19386 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19387 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19389 let mut goodbyes = Vec::new();
19392 while let Ok(outbound) = plexus_rx.try_recv() {
19393 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
19394 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
19395 }
19396 assert!(
19397 goodbyes.contains(&(bind_channel, bind_epoch)),
19398 "{goodbyes:?}"
19399 );
19400 assert!(rig
19401 .handler
19402 .registry
19403 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
19404 .unwrap()
19405 .is_some());
19406 }
19407 assert_eq!(ended, Some(true));
19408 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
19411 }
19412
19413 #[tokio::test]
19416 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
19417 struct Case {
19418 name: &'static str,
19419 change: fn(&mut ScopeRecord),
19420 closed: [Option<RouteCloseReason>; 4],
19423 }
19424 use RouteCloseReason::*;
19425 let cases = [
19426 Case {
19427 name: "a carrier entry removed",
19428 change: |r| {
19429 r.carriers.retain(|c| {
19430 c.principal
19431 != Principal::Reserved {
19432 module_id: AFT.to_string(),
19433 }
19434 })
19435 },
19436 closed: [None, Some(ScopeCarrierRemoved), None, None],
19437 },
19438 Case {
19439 name: "a target removed from a carrier",
19440 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
19441 closed: [None, None, None, Some(ScopeCarrierRemoved)],
19442 },
19443 Case {
19444 name: "a bare carrier narrowed to targets",
19445 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
19446 closed: [None, Some(ScopeCarrierRemoved), None, None],
19447 },
19448 Case {
19449 name: "delegates turned off",
19450 change: |r| r.attributes.delegates = false,
19451 closed: [Some(ScopeDelegationChanged); 4],
19452 },
19453 Case {
19454 name: "agent_id changed",
19455 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
19456 closed: [Some(ScopeDelegationChanged); 4],
19457 },
19458 Case {
19459 name: "a carrier added, child owners changed, the record re-sent",
19460 change: |r| {
19461 r.carriers.push(carrier(MAGIC, None));
19462 r.child_owners.push(Principal::Reserved {
19463 module_id: MAGIC.to_string(),
19464 });
19465 },
19466 closed: [None; 4],
19467 },
19468 Case {
19469 name: "a target added",
19470 change: |r| {
19471 r.carriers[1]
19472 .targets
19473 .as_mut()
19474 .unwrap()
19475 .push("third".to_string())
19476 },
19477 closed: [None; 4],
19478 },
19479 Case {
19480 name: "delegates turned on",
19481 change: |r| r.attributes.delegates = true,
19482 closed: [None; 4],
19483 },
19484 ];
19485 for case in cases {
19486 let mut rig = rig().await;
19487 rig.sync(vec![session(1)]).await;
19488 let scope = || Some(rig_selector("s", Some(1)));
19489 let mut routes = [
19490 rig.bound(Some(OWNER), PLEXUS, scope()).await,
19491 rig.bound(Some(AFT), PLEXUS, scope()).await,
19492 rig.bound(Some(BROCA), PLEXUS, scope()).await,
19493 rig.bound(Some(BROCA), OTHER, scope()).await,
19494 ];
19495 let mut record = session(1);
19496 (case.change)(&mut record);
19497 rig.sync(vec![record]).await;
19498 for (index, expected) in case.closed.iter().enumerate() {
19499 let route = &mut routes[index];
19500 match expected {
19501 Some(reason) => {
19502 assert!(!rig.live(route), "{}: route {index} still live", case.name);
19503 assert_eq!(
19504 route.closed_reason(),
19505 *reason,
19506 "{}: route {index}",
19507 case.name
19508 );
19509 }
19510 None => {
19511 assert!(rig.live(route), "{}: route {index} closed", case.name);
19512 assert!(
19513 route.untouched(),
19514 "{}: route {index} was told something",
19515 case.name
19516 );
19517 }
19518 }
19519 }
19520 }
19521 }
19522
19523 #[tokio::test]
19524 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
19525 for next in [Vec::new(), vec![session(2)]] {
19527 let mut rig = rig().await;
19528 rig.sync(vec![session(1)]).await;
19529 let mut route = rig
19530 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19531 .await;
19532 rig.sync(next).await;
19533 assert!(!rig.live(&route));
19534 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
19535 }
19536
19537 let mut rig = rig().await;
19540 let mut child = session(1);
19541 child.scope_ref = "child".to_string();
19542 child.kind = ScopeKind::Worker;
19543 child.parent = Some(ScopeParent::new(
19544 Principal::Reserved {
19545 module_id: OWNER.to_string(),
19546 },
19547 "s".to_string(),
19548 1,
19549 ));
19550 rig.sync(vec![session(1), child.clone()]).await;
19551 let mut child_route = rig
19552 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
19553 .await;
19554 assert_eq!(
19555 child_route.stamp().unwrap().parent_state,
19556 Some(ParentState::Linked)
19557 );
19558 rig.sync(vec![child]).await;
19559 assert!(!rig.live(&child_route));
19560 assert_eq!(
19561 child_route.closed_reason(),
19562 RouteCloseReason::ScopeParentEnded
19563 );
19564 }
19565
19566 #[tokio::test]
19567 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
19568 ) {
19569 let mut rig = rig().await;
19570 rig.sync(vec![session(1)]).await;
19571 let mut route = rig
19572 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19573 .await;
19574 let before = rig.forwarding.published_scope_tag(OWNER, "s");
19575 rig.sync(vec![session(1)]).await;
19576 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
19577 assert!(rig.live(&route) && route.untouched());
19578
19579 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
19583 .forwarding
19584 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
19585 .unwrap()
19586 else {
19587 panic!("the route is bound");
19588 };
19589 binding.flow.acquire_tagged(9, false).await.unwrap();
19590 let mut widened = session(1);
19591 widened.carriers.push(carrier(MAGIC, None));
19592 rig.sync(vec![widened]).await;
19593 assert!(rig.live(&route) && route.untouched());
19594 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19595 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
19596 assert_eq!(binding.flow.in_flight(), 1);
19599 binding
19600 .flow
19601 .acquire_tagged(10, false)
19602 .await
19603 .expect("the flow is still open");
19604 }
19605
19606 #[tokio::test]
19609 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
19610 let mut rig = rig().await;
19611 rig.sync(vec![session(1)]).await;
19612 let mut on_incumbent = rig
19613 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19614 .await;
19615
19616 let (candidate, _candidate_rx) = wide_ctx(9);
19619 let registration = rig
19620 .handler
19621 .registry
19622 .register_candidate_with_control_ops(
19623 manifest(PLEXUS, PROTOCOL_VERSION),
19624 PROTOCOL_VERSION,
19625 candidate.connection_id,
19626 module_baseline_control_ops(),
19627 )
19628 .unwrap();
19629 rig.forwarding
19630 .register_candidate_module_connection(
19631 candidate.connection_id,
19632 PLEXUS.to_string(),
19633 PROTOCOL_VERSION,
19634 manifest_concurrency(®istration.manifest),
19635 candidate.egress.clone(),
19636 )
19637 .unwrap();
19638 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
19639 rig.handler
19640 .registry
19641 .promote_candidate(PLEXUS)
19642 .unwrap()
19643 .unwrap();
19644 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
19645
19646 rig.sync(Vec::new()).await;
19647 assert!(!rig.live(&on_incumbent));
19648 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
19649 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19650 let goodbye = incumbent_rx
19651 .try_recv()
19652 .expect("the superseded endpoint is told")
19653 .frame;
19654 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19655 }
19656 }
19657}
19658
19659#[cfg(test)]
19660mod concurrency_default_exposure_tests {
19661 use super::*;
19662
19663 fn hello_body(role_json: &str) -> Vec<u8> {
19664 format!(
19665 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
19666 )
19667 .into_bytes()
19668 }
19669
19670 fn manifest_from(body: &[u8]) -> ModuleManifest {
19671 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
19672 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
19673 .expect("manifest parses")
19674 }
19675
19676 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
19677
19678 #[test]
19679 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
19680 let body = hello_body(&format!(
19681 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
19682 ));
19683 let manifest = manifest_from(&body);
19684 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
19687 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
19688 }
19689
19690 #[test]
19691 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
19692 let body = hello_body(&format!(
19693 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
19694 ));
19695 let manifest = manifest_from(&body);
19696 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
19697 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19698 }
19699
19700 #[test]
19701 fn non_management_roles_are_never_reported() {
19702 let body = hello_body(
19703 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
19704 );
19705 let manifest = manifest_from(&body);
19706 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19707 }
19708}