1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
26 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
27 },
28 scope::{
29 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
30 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
31 SCOPE_SYNC_OP,
32 },
33 session::{
34 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
35 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
36 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
37 ROLE_VERSIONS_FIELD,
38 },
39 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
40 Priority, RouteTarget, PROTOCOL_VERSION,
41};
42use tokio::time::{timeout_at, Instant};
43use tracing::{debug, info, warn};
44
45use crate::{
46 caller_identity::CallerIdentity,
47 capability_requirements::{
48 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
49 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
50 RuntimeModule,
51 },
52 child_roster::DaemonShutdownFlag,
53 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
54 forwarding::{
55 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
56 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
57 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
58 },
59 observability::{
60 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
61 },
62 privileged_prompt::{prompt_text, PrivilegedOperation},
63 provenance::{
64 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
65 },
66 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
67 router::{RouteCtx, RouterError},
68 scopes::{HelloLaunchNonces, ScopeTable},
69 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
70 stderr_tail::{CaptureState, TailEntry},
71 supervise::{
72 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
73 SupervisorHandle, SwapHelloAdmission,
74 },
75 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
76};
77
78pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
84
85const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
86const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
87const CAP_PING_PONG: &str = "ping_pong_v1";
88const CAP_SESSION_ATTACH: &str = "session_attach_v1";
89const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
90
91const SUBC_CONTROL_OPS: &[&str] = &[
92 ops::ACTIVITY_RECORD,
93 ops::ACTIVITY_QUERY,
94 ops::ACTIVITY_SUBSCRIBE,
95 ops::SERVER_DESCRIBE,
96 ops::CATALOG_LIST,
97 ops::ROUTE_OPEN,
98 ops::ROUTE_POLL,
99 ops::ROUTE_CLOSING,
100 ops::ROUTE_CLOSED,
101 ops::SUPERVISOR_LIST,
102 ops::SUPERVISOR_RESTART,
103 ops::SUPERVISOR_SWAP,
104 ops::SUPERVISOR_RELOAD,
105 ops::SUPERVISOR_RESCAN,
106 ops::SUPERVISOR_RELEASE_RESERVED,
107 ops::SUPERVISOR_SET_ENABLED,
108 ops::SUPERVISOR_HEALTH_PROBE,
109 ops::SUPERVISOR_HEALTH,
110 ops::SUPERVISOR_STDERR_TAIL,
111 ops::SUPERVISOR_TERMINALS,
112 ops::SUPERVISOR_ROUTES,
113 ops::SUPERVISOR_PROVENANCE,
114 ops::SUPERVISOR_SPAWN_SNAPSHOT,
115 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
116];
117
118const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
119 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
120 "supervisor.live_roots",
121 SCOPE_SYNC_OP,
122 SCOPE_APPLY_OP,
123 SCOPE_DESCRIBE_OP,
124 "operator.confirm",
125];
126
127const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
131
132const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
133
134pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
142
143pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
158
159pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
177
178const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
179const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
180
181fn reload_verdict(
182 configured: &Path,
183 spawned_from: Option<&Path>,
184 image: subc_control::RunningImageAgreement,
185) -> PendingReloadVerdict {
186 let path = match spawned_from {
187 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
188 ReloadPathAgreement::Match
189 }
190 Some(spawned_from) => ReloadPathAgreement::Mismatch {
191 configured: configured.to_path_buf(),
192 spawned_from: spawned_from.to_path_buf(),
193 },
194 None => ReloadPathAgreement::Unavailable {
195 reason: if matches!(
196 image,
197 subc_control::RunningImageAgreement::Unavailable {
198 reason: subc_control::RunningImageUnavailableReason::NotRunning
199 }
200 ) {
201 ReloadPathUnavailableReason::NotRunning
202 } else {
203 ReloadPathUnavailableReason::SpawnedPathUnavailable
204 },
205 },
206 };
207 PendingReloadVerdict {
208 path,
209 image,
210 spec: None,
211 }
212}
213
214fn launch_spec_agreement(
215 configured: &crate::supervise::ModuleSpec,
216 running: bool,
217 spawned: Option<&crate::supervise::LaunchSpecDigest>,
218) -> subc_control::LaunchSpecAgreement {
219 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
220 if !running {
221 return Agreement::Unavailable {
222 reason: Reason::NotRunning,
223 };
224 }
225 match spawned {
226 None => Agreement::Unavailable {
227 reason: Reason::SpawnedSpecUnavailable,
228 },
229 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
230 Agreement::Match
231 }
232 Some(_) => Agreement::Mismatch,
233 }
234}
235
236fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
237 if configured == spawned_from {
238 return true;
239 }
240 #[cfg(windows)]
241 {
242 match (
248 std::fs::canonicalize(configured),
249 std::fs::canonicalize(spawned_from),
250 ) {
251 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
252 _ => false,
253 }
254 }
255 #[cfg(not(windows))]
256 {
257 false
258 }
259}
260
261#[derive(Clone)]
262struct DaemonProvenanceFacts {
263 build: DaemonBuildProvenance,
264 pid: Option<u32>,
265 started_at_ms: Option<u64>,
266 start_clock: Option<crate::clock::StartClock>,
267 executable_path: Option<PathBuf>,
268 executable_identity: Option<SpawnedFileIdentity>,
269 process_start_time: Option<u64>,
270 probe: ExecutableIdentityProbe,
271}
272
273impl Default for DaemonProvenanceFacts {
274 fn default() -> Self {
275 Self {
276 build: DaemonBuildProvenance {
277 build_git_sha: None,
278 build_lock_digest: None,
279 },
280 pid: None,
281 started_at_ms: None,
282 start_clock: None,
283 executable_path: None,
284 executable_identity: None,
285 process_start_time: None,
286 probe: ExecutableIdentityProbe::default(),
287 }
288 }
289}
290
291#[derive(Debug, Clone)]
292struct SupervisorRescanContext {
293 supervisor: Supervisor,
294 config_path: PathBuf,
295 configured_port: Option<u16>,
296 storage_config: Option<crate::daemon_config::StorageConfig>,
297 admission_facts_carrier_module_id: Option<String>,
298 admission_facts_targets: Option<Vec<String>>,
299 scope_authority_owners: Vec<String>,
300 privileged_operator_prompt: PrivilegedOperatorPrompt,
301}
302
303const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
324 "reloading",
325 "supervisor_not_live",
326 "registration_not_active",
327 "no_forwarding_connection",
328 "relay_send_failed",
329];
330
331#[derive(Clone)]
333pub struct ControlHandler {
334 registry: Arc<Registry>,
335 activity: Option<crate::activity::ActivityLedger>,
336 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
337 forwarding: Arc<ForwardingTable>,
338 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
339 supervisor: SupervisorHandle,
340 subc_capabilities: Arc<[String]>,
341 route_bind_relay_timeout: Duration,
345 route_bind_relay_timeouts: BTreeMap<String, Duration>,
349 route_bind_breakers: RouteBindBreakers,
352 route_bind_concurrency: RouteBindConcurrency,
355 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
360 route_bind_breaker_threshold: u32,
362 route_bind_breaker_cooldown: Duration,
364 health_probe_timeout: Duration,
365 storage_config: Option<crate::daemon_config::StorageConfig>,
368 machine_id: Option<crate::machine_id::MachineId>,
372 admission_facts_carrier_module_id: Option<String>,
373 admission_facts_targets: Option<Vec<String>>,
374 scopes: Arc<RwLock<ScopeTable>>,
378 scope_deadline_added: Arc<tokio::sync::Notify>,
382 scope_authority_owners: Vec<String>,
385 privileged_operator_prompt: PrivilegedOperatorPrompt,
387 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
390 #[cfg(test)]
391 caller_lookup: Option<TestCallerLookup>,
392 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
395 rescan: Option<SupervisorRescanContext>,
396 connected_clients: ConnectedClients,
397 counters: DaemonCounters,
398 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
399 daemon_shutdown: DaemonShutdownFlag,
404 daemon_provenance: DaemonProvenanceFacts,
405 #[cfg(test)]
406 control_dispatch_delay: Option<Duration>,
407 #[cfg(test)]
408 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
409 #[cfg(test)]
413 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
414 #[cfg(test)]
420 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
421}
422
423impl fmt::Debug for ControlHandler {
424 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
425 f.debug_struct("ControlHandler")
426 .field("registry", &self.registry)
427 .field("forwarding", &self.forwarding)
428 .field("process_liveness", &self.process_liveness.is_some())
429 .field("supervisor", &self.supervisor)
430 .field("subc_capabilities", &self.subc_capabilities)
431 .finish()
432 }
433}
434
435struct RouteOpenRequest {
436 target: RouteTarget,
437 identity: BindIdentity,
438 consumer_identity: Option<ConsumerIdentity>,
439 consumer_capabilities: Option<Vec<String>>,
440 role_versions: Option<BTreeMap<String, String>>,
441 admission_facts: Option<serde_json::Value>,
442 scope: Option<ScopeSelector>,
443}
444
445#[derive(Clone, Copy)]
446enum RouteOpenPhase {
447 Admission,
448 ProjectRoot,
449 EgressReservation,
450 BindSend,
451 BindAckWait,
452}
453
454struct RouteOpenTimings {
458 module_id: String,
459 principal: String,
460 admission: Duration,
461 project_root: Duration,
462 egress_reservation: Duration,
463 bind_send: Duration,
464 bind_ack_wait: Duration,
465 phase: Option<RouteOpenPhase>,
466 phase_started_at: Instant,
467}
468
469impl RouteOpenTimings {
470 fn new() -> Self {
471 Self {
472 module_id: String::new(),
473 principal: "unresolved".to_string(),
475 admission: Duration::ZERO,
476 project_root: Duration::ZERO,
477 egress_reservation: Duration::ZERO,
478 bind_send: Duration::ZERO,
479 bind_ack_wait: Duration::ZERO,
480 phase: Some(RouteOpenPhase::Admission),
481 phase_started_at: Instant::now(),
482 }
483 }
484
485 fn finish(&mut self) {
486 let elapsed = self.phase_started_at.elapsed();
487 match self.phase.take() {
488 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
489 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
490 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
491 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
492 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
493 None => {}
494 }
495 }
496
497 fn enter(&mut self, phase: RouteOpenPhase) {
498 self.finish();
499 self.phase = Some(phase);
500 self.phase_started_at = Instant::now();
501 }
502}
503
504struct RouteBindReservationGuard {
505 forwarding: Arc<ForwardingTable>,
506 endpoint: ModuleEndpointId,
507 relay_corr: u64,
508 armed: bool,
509}
510
511struct ModuleControlRpcGuard {
512 forwarding: Arc<ForwardingTable>,
513 endpoint: ModuleEndpointId,
514 corr: u64,
515 armed: bool,
516}
517
518impl ModuleControlRpcGuard {
519 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
520 Self {
521 forwarding,
522 endpoint,
523 corr,
524 armed: true,
525 }
526 }
527
528 fn disarm(&mut self) {
529 self.armed = false;
530 }
531}
532
533impl Drop for ModuleControlRpcGuard {
534 fn drop(&mut self) {
535 if self.armed {
536 let _ = self
537 .forwarding
538 .cancel_module_control_rpc(self.endpoint, self.corr);
539 }
540 }
541}
542
543impl RouteBindReservationGuard {
544 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
545 Self {
546 forwarding,
547 endpoint,
548 relay_corr,
549 armed: true,
550 }
551 }
552
553 fn release_and_disarm(&mut self) {
554 if !self.armed {
555 return;
556 }
557 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
558 self.endpoint,
559 self.relay_corr,
560 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
561 ) {
562 send_goodbye_target_best_effort(
563 &self.forwarding.counters(),
564 &target,
565 "canceled route.bind",
566 );
567 }
568 self.armed = false;
569 }
570
571 fn disarm(&mut self) {
572 self.armed = false;
573 }
574}
575
576impl Drop for RouteBindReservationGuard {
577 fn drop(&mut self) {
578 self.release_and_disarm();
579 }
580}
581
582#[derive(Debug, Clone, Default)]
612pub(crate) struct RouteBindBreakers {
613 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
614}
615
616#[derive(Debug, Clone, Default)]
617pub(crate) struct RouteBindConcurrency {
618 modules: Arc<Mutex<HashMap<String, usize>>>,
619}
620
621struct RouteBindConcurrencyGuard {
622 concurrency: RouteBindConcurrency,
623 module_id: String,
624}
625
626impl RouteBindConcurrency {
627 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
630 let mut modules = self
631 .modules
632 .lock()
633 .expect("route.bind concurrency mutex poisoned");
634 let in_flight = modules.entry(module_id.to_string()).or_default();
635 if *in_flight >= limit {
636 return Err(*in_flight);
637 }
638 *in_flight += 1;
639 Ok(RouteBindConcurrencyGuard {
640 concurrency: self.clone(),
641 module_id: module_id.to_string(),
642 })
643 }
644}
645
646impl Drop for RouteBindConcurrencyGuard {
647 fn drop(&mut self) {
648 let mut modules = self
649 .concurrency
650 .modules
651 .lock()
652 .expect("route.bind concurrency mutex poisoned");
653 let remove = {
654 let in_flight = modules
655 .get_mut(&self.module_id)
656 .expect("admitted route.bind has a concurrency entry");
657 *in_flight -= 1;
658 *in_flight == 0
659 };
660 if remove {
661 modules.remove(&self.module_id);
662 }
663 }
664}
665
666#[derive(Debug, Default)]
667struct ModuleBreakerState {
668 consecutive_timeouts: u32,
670 cooldown_until: Option<Instant>,
673 probe_in_flight: Option<Arc<()>>,
678}
679
680enum RouteBindAdmission<'a> {
682 Admitted {
683 guard: RouteBindBreakerGuard<'a>,
684 probe: bool,
687 },
688 Refused {
689 consecutive_timeouts: u32,
690 retry_in: Duration,
694 probe_in_flight: bool,
695 },
696}
697
698struct RouteBindBreakerGuard<'a> {
705 breakers: RouteBindBreakers,
706 module_id: &'a str,
707 probe_token: Option<Arc<()>>,
708 settled: bool,
709}
710
711impl RouteBindBreakerGuard<'_> {
712 fn record_accepted(&mut self) -> bool {
716 self.settled = true;
717 self.breakers.record_accepted(self.module_id)
718 }
719
720 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
723 self.settled = true;
724 self.breakers.record_timeout(
725 self.module_id,
726 self.probe_token.as_ref(),
727 threshold,
728 cooldown,
729 )
730 }
731
732 fn record_inconclusive(&mut self) {
741 self.settled = true;
742 self.breakers
743 .record_inconclusive(self.module_id, self.probe_token.as_ref());
744 }
745}
746
747impl Drop for RouteBindBreakerGuard<'_> {
748 fn drop(&mut self) {
749 if !self.settled {
750 self.breakers
751 .record_inconclusive(self.module_id, self.probe_token.as_ref());
752 }
753 }
754}
755
756struct BreakerOpened {
760 consecutive_timeouts: u32,
761 reopened_after_probe: bool,
764}
765
766impl RouteBindBreakers {
767 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
768 self.modules
769 .lock()
770 .expect("route.bind breaker mutex poisoned")
771 }
772
773 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
776 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
777 probe: probe_token.is_some(),
778 guard: RouteBindBreakerGuard {
779 breakers: self.clone(),
780 module_id,
781 probe_token,
782 settled: false,
783 },
784 };
785
786 let mut modules = self.lock();
787 let Some(state) = modules.get_mut(module_id) else {
788 return admitted(None);
789 };
790 let Some(cooldown_until) = state.cooldown_until else {
791 return admitted(None);
792 };
793 if state.probe_in_flight.is_some() {
794 return RouteBindAdmission::Refused {
795 consecutive_timeouts: state.consecutive_timeouts,
796 retry_in: Duration::ZERO,
797 probe_in_flight: true,
798 };
799 }
800 let now = Instant::now();
801 if now < cooldown_until {
802 return RouteBindAdmission::Refused {
803 consecutive_timeouts: state.consecutive_timeouts,
804 retry_in: cooldown_until - now,
805 probe_in_flight: false,
806 };
807 }
808 let token = Arc::new(());
809 state.probe_in_flight = Some(Arc::clone(&token));
810 admitted(Some(token))
811 }
812
813 fn record_accepted(&self, module_id: &str) -> bool {
814 self.lock()
815 .remove(module_id)
816 .is_some_and(|state| state.cooldown_until.is_some())
817 }
818
819 fn record_timeout(
820 &self,
821 module_id: &str,
822 probe_token: Option<&Arc<()>>,
823 threshold: u32,
824 cooldown: Duration,
825 ) -> Option<BreakerOpened> {
826 let mut modules = self.lock();
827 let state = modules.entry(module_id.to_string()).or_default();
828 let was_open = state.cooldown_until.is_some();
829 let was_probe = Self::owns_probe(state, probe_token);
830 if was_probe {
831 state.probe_in_flight = None;
832 }
833 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
834 if state.consecutive_timeouts < threshold {
835 return None;
836 }
837 state.cooldown_until = Some(Instant::now() + cooldown);
838 Some(BreakerOpened {
839 consecutive_timeouts: state.consecutive_timeouts,
840 reopened_after_probe: was_open && was_probe,
841 })
842 }
843
844 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
845 state
849 .probe_in_flight
850 .as_ref()
851 .zip(token)
852 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
853 }
854
855 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
856 if let Some(state) = self.lock().get_mut(module_id) {
857 if Self::owns_probe(state, probe_token) {
858 state.probe_in_flight = None;
859 }
860 }
861 }
862
863 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
878 self.lock()
879 .remove(module_id)
880 .map(|state| state.consecutive_timeouts)
881 .filter(|discarded| *discarded > 0)
882 }
883
884 fn open_snapshot(&self) -> Option<serde_json::Value> {
891 let now = Instant::now();
892 let modules = self.lock();
893 let open = modules
894 .iter()
895 .filter_map(|(module_id, state)| {
896 let cooldown_until = state.cooldown_until?;
897 Some((
898 module_id.clone(),
899 serde_json::json!({
900 "consecutive_timeouts": state.consecutive_timeouts,
901 "cooldown_remaining_ms":
902 cooldown_until.saturating_duration_since(now).as_millis() as u64,
903 "probe_in_flight": state.probe_in_flight.is_some(),
904 }),
905 ))
906 })
907 .collect::<serde_json::Map<String, serde_json::Value>>();
908 (!open.is_empty()).then_some(serde_json::Value::Object(open))
909 }
910}
911
912impl ControlHandler {
913 pub fn new(registry: Arc<Registry>) -> Self {
914 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
915 }
916
917 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
918 let counters = forwarding.counters();
919 let route_bind_breakers = forwarding.route_bind_breakers();
923 let route_bind_concurrency = forwarding.route_bind_concurrency();
924 let route_outages = forwarding.route_outages();
925 Self {
926 registry,
927 activity: None,
928 activity_streams: Arc::new(Mutex::new(HashMap::new())),
929 forwarding,
930 process_liveness: None,
931 supervisor: SupervisorHandle::new(),
932 subc_capabilities: Arc::from([
933 CAP_MANIFEST_REGISTRATION.to_string(),
934 CAP_CHANNEL_LIFECYCLE.to_string(),
935 CAP_PING_PONG.to_string(),
936 CAP_SESSION_ATTACH.to_string(),
937 CAP_ADMISSION_FACTS_RELAY.to_string(),
938 CAP_SCOPES_V1.to_string(),
939 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
940 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
941 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
942 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
943 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
944 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
945 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
946 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
947 ]),
948 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
949 route_bind_relay_timeouts: BTreeMap::new(),
950 route_bind_breakers,
951 route_bind_concurrency,
952 route_outages,
953 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
954 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
955 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
956 storage_config: None,
957 machine_id: None,
958 admission_facts_carrier_module_id: None,
959 admission_facts_targets: None,
960 scopes: Arc::new(RwLock::new(ScopeTable::new(
961 crate::daemon_config::default_scope_authority_owners(),
962 ))),
963 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
964 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
965 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
966 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
967 #[cfg(test)]
968 caller_lookup: None,
969 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
970 rescan: None,
971 connected_clients: ConnectedClients::new(),
972 counters,
973 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
974 daemon_shutdown: DaemonShutdownFlag::default(),
975 daemon_provenance: DaemonProvenanceFacts::default(),
976 #[cfg(test)]
977 control_dispatch_delay: None,
978 #[cfg(test)]
979 provenance_probe_override: None,
980 #[cfg(test)]
981 route_bind_relay_sent_hold: None,
982 #[cfg(test)]
983 hello_registry_to_forwarding_hold: None,
984 }
985 }
986
987 pub fn with_storage_config(
990 mut self,
991 storage_config: Option<crate::daemon_config::StorageConfig>,
992 ) -> Self {
993 self.storage_config = storage_config;
994 self
995 }
996
997 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1000 self.activity = Some(crate::activity::ActivityLedger::open(
1001 run_dir.join("activity.jsonl"),
1002 incarnation,
1003 ));
1004 self.supervisor.set_activity(self.activity.clone());
1005 self
1006 }
1007
1008 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1011 self.machine_id = machine_id;
1012 self
1013 }
1014
1015 pub fn with_admission_facts_config(
1019 mut self,
1020 carrier_module_id: Option<String>,
1021 targets: Option<Vec<String>>,
1022 ) -> Self {
1023 self.admission_facts_carrier_module_id = carrier_module_id;
1024 self.admission_facts_targets = targets;
1025 self
1026 }
1027
1028 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1032 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1033 self.scope_authority_owners = owners;
1034 self
1035 }
1036
1037 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1039 self.privileged_operator_prompt = mode;
1040 self
1041 }
1042
1043 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1046 self.route_bind_relay_timeout = timeout;
1047 self
1048 }
1049
1050 pub fn with_route_bind_relay_timeouts(
1056 mut self,
1057 timeouts: impl IntoIterator<Item = (String, Duration)>,
1058 ) -> Self {
1059 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1060 self
1061 }
1062
1063 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1069 self.route_bind_relay_timeouts
1070 .get(module_id)
1071 .copied()
1072 .unwrap_or(self.route_bind_relay_timeout)
1073 }
1074
1075 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1083 self.route_bind_breaker_threshold = threshold.max(1);
1084 self.route_bind_breaker_cooldown = cooldown;
1085 self
1086 }
1087
1088 #[cfg(test)]
1089 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1090 self.health_probe_timeout = timeout;
1091 self
1092 }
1093
1094 #[cfg(test)]
1095 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1096 self.control_dispatch_delay = Some(delay);
1097 self
1098 }
1099
1100 #[cfg(test)]
1104 pub(crate) fn with_route_bind_relay_sent_hold(
1105 mut self,
1106 hold: Arc<tokio::sync::Barrier>,
1107 ) -> Self {
1108 self.route_bind_relay_sent_hold = Some(hold);
1109 self
1110 }
1111
1112 #[cfg(test)]
1117 pub(crate) fn with_hello_registry_to_forwarding_hold(
1118 mut self,
1119 hold: Arc<std::sync::Barrier>,
1120 ) -> Self {
1121 self.hello_registry_to_forwarding_hold = Some(hold);
1122 self
1123 }
1124
1125 #[cfg(test)]
1126 fn hold_between_registry_and_forwarding(&self) {
1127 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1128 hold.wait();
1129 hold.wait();
1130 }
1131 }
1132
1133 pub fn with_process_liveness(
1134 mut self,
1135 process_liveness: Arc<dyn ModuleProcessLiveness>,
1136 ) -> Self {
1137 self.process_liveness = Some(process_liveness);
1138 self
1139 }
1140
1141 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1142 self.supervisor = supervisor;
1143 self.supervisor.set_activity(self.activity.clone());
1144 self
1145 }
1146
1147 pub fn with_daemon_provenance(
1148 mut self,
1149 pid: u32,
1150 started_at_ms: u64,
1151 executable_path: Option<PathBuf>,
1152 build_git_sha: Option<String>,
1153 build_lock_digest: Option<String>,
1154 ) -> Self {
1155 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1156 let process_start_time = process_start_time(pid);
1157 self.daemon_provenance = DaemonProvenanceFacts {
1158 build: DaemonBuildProvenance {
1159 build_git_sha,
1160 build_lock_digest,
1161 },
1162 pid: Some(pid),
1163 started_at_ms: Some(started_at_ms),
1164 start_clock: None,
1165 executable_path,
1166 executable_identity,
1167 process_start_time,
1168 probe: ExecutableIdentityProbe::default(),
1169 };
1170 self
1171 }
1172
1173 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1174 self.daemon_provenance.start_clock = Some(clock);
1175 self
1176 }
1177
1178 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1180 self.scopes
1181 .write()
1182 .unwrap_or_else(|poisoned| poisoned.into_inner())
1183 .set_wall_clock(Arc::new(clock));
1184 self
1185 }
1186
1187 #[cfg(test)]
1188 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1189 self.provenance_probe_override = Some(result);
1190 self
1191 }
1192
1193 pub fn with_capability_config(
1197 self,
1198 modules: impl IntoIterator<Item = (String, bool)>,
1199 reserved_capabilities: BTreeMap<String, String>,
1200 ) -> Self {
1201 self.capability_evaluator
1202 .configure(modules, reserved_capabilities);
1203 self
1204 }
1205
1206 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1210 self.daemon_shutdown = flag;
1211 self
1212 }
1213
1214 pub fn with_supervisor_rescan(
1215 mut self,
1216 supervisor: Supervisor,
1217 config_path: impl Into<PathBuf>,
1218 configured_port: Option<u16>,
1219 ) -> Self {
1220 self.rescan = Some(SupervisorRescanContext {
1221 supervisor,
1222 config_path: config_path.into(),
1223 configured_port,
1224 storage_config: self.storage_config.clone(),
1225 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1226 admission_facts_targets: self.admission_facts_targets.clone(),
1227 scope_authority_owners: self.scope_authority_owners.clone(),
1228 privileged_operator_prompt: self.privileged_operator_prompt,
1229 });
1230 self
1231 }
1232
1233 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1234 self.connected_clients = connected_clients;
1235 self
1236 }
1237
1238 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1239 Arc::clone(&self.forwarding)
1240 }
1241
1242 pub(crate) fn counters(&self) -> DaemonCounters {
1243 self.counters.clone()
1244 }
1245
1246 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1249 tokio::spawn(async move {
1250 loop {
1251 self.capability_evaluator
1252 .wait_for_change_or_deadline()
1253 .await;
1254 self.refresh_capability_requirements();
1255 }
1256 });
1257 }
1258
1259 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1265 tokio::spawn(async move {
1266 loop {
1267 let has_deadlines = self
1268 .scopes
1269 .read()
1270 .unwrap_or_else(|poisoned| poisoned.into_inner())
1271 .has_deadlines();
1272 if !has_deadlines {
1273 self.scope_deadline_added.notified().await;
1274 continue;
1275 }
1276 tokio::time::sleep(Duration::from_secs(1)).await;
1277 if let Err(error) = self.sweep_expired_scopes() {
1278 warn!(%error, "scope expiry sweep failed");
1279 }
1280 }
1281 });
1282 }
1283
1284 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1285 let mut table = self
1286 .scopes
1287 .write()
1288 .unwrap_or_else(|poisoned| poisoned.into_inner());
1289 let swept = table.sweep_expired();
1290 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1291 drop(table);
1292 self.close_scope_drained_routes(drained);
1293 Ok(())
1294 }
1295
1296 fn publish_scope_changes(
1297 &self,
1298 changes: &[crate::scopes::ScopeTagChange],
1299 expired: &[crate::scopes::ScopeExpired],
1300 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1301 if changes.is_empty() {
1302 return Ok(Vec::new());
1303 }
1304 let drained = self
1305 .forwarding
1306 .publish_scope_changes(changes)
1307 .map_err(RouterError::Forwarding)?;
1308 let mut counts = HashMap::new();
1309 for route in &drained {
1310 *counts
1311 .entry((
1312 route.scope.owner.as_str(),
1313 route.scope.scope_ref.as_str(),
1314 route.scope.tag.scope_epoch,
1315 ))
1316 .or_insert(0usize) += 1;
1317 }
1318 for scope in expired {
1319 let routes_closed = counts
1320 .get(&(
1321 scope.owner.as_str(),
1322 scope.scope_ref.as_str(),
1323 scope.scope_epoch,
1324 ))
1325 .copied()
1326 .unwrap_or(0);
1327 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1328 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1329 routes_closed, "scope expired");
1330 }
1331 Ok(drained)
1332 }
1333
1334 fn runtime_capability_snapshot(
1335 &self,
1336 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1337 let runtime = self
1338 .supervisor
1339 .list()
1340 .into_iter()
1341 .map(|module| {
1342 let status = module.status().map_err(|err| {
1343 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1344 })?;
1345 Ok(RuntimeModule {
1346 module_id: status.module_id,
1347 state: status.state,
1348 enabled: status.enabled,
1349 })
1350 })
1351 .collect::<Result<Vec<_>, RouterError>>()?;
1352 let registrations = self.registry.list_registered_modules().map_err(|err| {
1358 RouterError::backend(
1359 0,
1360 0,
1361 format!("failed to list capability registrations: {err}"),
1362 )
1363 })?;
1364 let registrations = registrations
1365 .into_iter()
1366 .map(|registration| RegisteredModule {
1367 module_id: registration.manifest.module_id,
1368 module_version: registration.manifest.module_version,
1369 capabilities: registration.manifest.capabilities,
1370 })
1371 .collect();
1372 Ok((runtime, registrations))
1373 }
1374
1375 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1382 let cached_registration = RegisteredModule {
1383 module_id: registration.manifest.module_id.clone(),
1384 module_version: registration.manifest.module_version.clone(),
1385 capabilities: registration.manifest.capabilities.clone(),
1386 };
1387 if self.capability_evaluator.record_hello(&cached_registration) {
1388 warn!(
1389 module_id = %cached_registration.module_id,
1390 "capability claims drifted from the cached manifest"
1391 );
1392 }
1393 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1394 self.enforce_capability_denies();
1395 }
1396 self.refresh_capability_requirements();
1397 }
1398
1399 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1403 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1404 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1405 self.supervisor.set_swap_promotion_observer(observer);
1406 }
1407
1408 pub fn refresh_capability_requirements(&self) {
1409 match self.runtime_capability_snapshot() {
1410 Ok((runtime, registrations)) => {
1411 log_requirement_events(
1412 self.capability_evaluator
1413 .evaluate_now(&runtime, ®istrations),
1414 &self.daemon_shutdown,
1415 );
1416 }
1417 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1418 }
1419 }
1420
1421 fn enforce_capability_denies(&self) {
1425 let registrations = match self.registry.list_registered_modules() {
1426 Ok(snapshot) => snapshot,
1427 Err(err) => {
1428 warn!(error = %err, "failed to read registrations for capability deny census");
1429 return;
1430 }
1431 };
1432 let manifests = registrations
1433 .into_iter()
1434 .map(|registration| {
1435 (
1436 registration.manifest.module_id.clone(),
1437 registration.manifest,
1438 )
1439 })
1440 .collect::<BTreeMap<_, _>>();
1441 let census = match self.forwarding.route_census(None) {
1442 Ok(census) => census,
1443 Err(err) => {
1444 warn!(error = %err, "failed to read route census for capability deny enforcement");
1445 return;
1446 }
1447 };
1448
1449 for (target_module_id, routes) in census {
1450 let Some(target_manifest) = manifests.get(&target_module_id) else {
1451 continue;
1452 };
1453 let mut closed_routes = Vec::new();
1454 let mut module_goodbyes = Vec::new();
1455 for route in routes {
1456 let Principal::Reserved {
1457 module_id: opening_module_id,
1458 } = &route.principal
1459 else {
1460 continue;
1461 };
1462 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1463 continue;
1464 };
1465 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1466 continue;
1467 };
1468
1469 match self.forwarding.release_client_route(
1470 route.goodbye_target.connection_id,
1471 route.goodbye_target.channel,
1472 route.goodbye_target.epoch,
1473 ) {
1474 Ok(RouteRelease::Removed(module_goodbye)) => {
1475 warn!(
1476 opening_module_id,
1477 target_module_id,
1478 capability,
1479 "force-closing route because an attested capability deny edge now matches"
1480 );
1481 closed_routes.push(route);
1482 module_goodbyes.push(module_goodbye);
1483 }
1484 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1485 Err(err) => warn!(
1486 opening_module_id,
1487 target_module_id,
1488 capability,
1489 error = %err,
1490 "failed to force-close capability-denied route"
1491 ),
1492 }
1493 }
1494
1495 if closed_routes.is_empty() {
1496 continue;
1497 }
1498 send_route_control_pushes(
1499 &self.forwarding,
1500 closed_routes,
1501 ClientControlPush::RouteClosed {
1502 module_id: target_module_id,
1503 channels: Vec::new(),
1504 reason: RouteCloseReason::CapabilityDenied,
1505 drained: false,
1506 abandoned: 0,
1507 excluded_subscriptions: 0,
1508 terminal: Some(false),
1509 },
1510 );
1511 self.emit_route_goodbyes(module_goodbyes);
1512 }
1513 }
1514
1515 fn not_ready_reason(
1521 &self,
1522 registration: &crate::registry::ModuleRegistration,
1523 ) -> Option<NotReadyReason> {
1524 if !registration.ready {
1525 return Some(NotReadyReason {
1526 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1527 capability: None,
1528 });
1529 }
1530 self.first_unprovided_required_capability(registration)
1531 .map(|capability| NotReadyReason {
1532 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1533 capability: Some(capability),
1534 })
1535 }
1536
1537 fn first_unprovided_required_capability(
1555 &self,
1556 registration: &crate::registry::ModuleRegistration,
1557 ) -> Option<String> {
1558 let required = registration
1559 .manifest
1560 .capabilities
1561 .iter()
1562 .flat_map(|declarations| declarations.requires.iter())
1563 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1564 .map(|requirement| requirement.capability.as_str())
1565 .collect::<BTreeSet<_>>();
1566 if required.is_empty() {
1567 return None;
1568 }
1569 let module_id = registration.manifest.module_id.as_str();
1570 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1571 if required
1572 .iter()
1573 .any(|capability| verdict(capability).is_none())
1574 {
1575 self.refresh_capability_requirements();
1576 }
1577 required
1578 .into_iter()
1579 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1580 .map(str::to_string)
1581 }
1582
1583 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1584 self.capability_evaluator
1585 .statuses()
1586 .into_iter()
1587 .map(capability_requirement_status)
1588 .collect()
1589 }
1590
1591 fn deregister_connection(
1598 &self,
1599 connection_id: ConnectionId,
1600 reason: RegistrationEndReason,
1601 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1602 self.registry
1603 .deregister_connection_with_reason(connection_id, reason)
1604 }
1605
1606 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1607 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1608 return None;
1609 }
1610 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1611 parse_client_control_request(&frame.body)
1612 else {
1613 return None;
1614 };
1615 Some(target_module_id(&target).to_string())
1616 }
1617
1618 pub(crate) fn route_open_capacity_refusal(
1619 &self,
1620 ctx: &RouteCtx,
1621 frame: &Frame,
1622 target_module_id: &str,
1623 in_flight: usize,
1624 limit: usize,
1625 ) -> Result<Frame, RouterError> {
1626 self.route_open_admission_refusal_frame(
1627 ctx,
1628 frame,
1629 target_module_id,
1630 "open_admission_full",
1631 (in_flight, limit),
1632 format!(
1633 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1634 ),
1635 )
1636 }
1637
1638 fn route_open_target_capacity_refusal(
1639 &self,
1640 ctx: &RouteCtx,
1641 frame: &Frame,
1642 target_module_id: &str,
1643 in_flight: usize,
1644 ) -> Result<Frame, RouterError> {
1645 self.route_open_admission_refusal_frame(
1646 ctx,
1647 frame,
1648 target_module_id,
1649 "target_binds_full",
1650 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1651 format!(
1652 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1653 ),
1654 )
1655 }
1656
1657 fn route_open_admission_refusal_frame(
1665 &self,
1666 ctx: &RouteCtx,
1667 frame: &Frame,
1668 target_module_id: &str,
1669 reason: &'static str,
1670 (in_flight, limit): (usize, usize),
1671 message: impl Into<String>,
1672 ) -> Result<Frame, RouterError> {
1673 let code = error_codes::TARGET_UNAVAILABLE;
1674 self.counters.increment_route_open_refused(code);
1675 info!(
1676 target: "control",
1677 code,
1678 reason,
1679 module_id = ?target_module_id,
1680 connection_id = ctx.connection_id.get(),
1681 in_flight,
1682 limit,
1683 "route.open refused"
1684 );
1685 control_error_frame(frame, code, message.into())
1686 }
1687
1688 #[cfg(test)]
1694 pub fn handle_control(
1695 &self,
1696 connection_id: ConnectionId,
1697 frame: Frame,
1698 ) -> Result<Vec<Frame>, RouterError> {
1699 match frame.header.ty {
1700 FrameType::Ping => Ok(vec![pong(&frame)?]),
1701 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1702 FrameType::Goodbye => self.handle_goodbye(connection_id),
1703 ty => Ok(vec![control_error_frame(
1704 &frame,
1705 "unsupported_control_frame",
1706 format!("unsupported channel-0 frame {ty:?}"),
1707 )?]),
1708 }
1709 }
1710
1711 pub async fn handle_control_frame(
1712 &self,
1713 ctx: &RouteCtx,
1714 frame: Frame,
1715 ) -> Result<Vec<Frame>, RouterError> {
1716 self.handle_control_frame_timed(ctx, frame, None).await
1717 }
1718
1719 pub(crate) fn record_caller_endpoints(
1720 &self,
1721 connection: ConnectionId,
1722 endpoints: Option<(SocketAddr, SocketAddr)>,
1723 ) {
1724 let mut callers = self
1725 .caller_endpoints
1726 .lock()
1727 .unwrap_or_else(|p| p.into_inner());
1728 callers.remove(&connection);
1729 if let Some(endpoints) = endpoints {
1730 callers.insert(connection, endpoints);
1731 }
1732 }
1733
1734 #[cfg(test)]
1735 pub(crate) fn with_caller_lookup_for_test(
1736 mut self,
1737 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1738 ) -> Self {
1739 self.caller_lookup = Some(Arc::new(lookup));
1740 self
1741 }
1742
1743 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1744 let endpoints = self
1745 .caller_endpoints
1746 .lock()
1747 .unwrap_or_else(|p| p.into_inner())
1748 .get(&connection)
1749 .copied();
1750 #[cfg(test)]
1751 let lookup = self.caller_lookup.clone();
1752 tokio::task::spawn_blocking(move || {
1753 #[cfg(test)]
1754 if let Some(lookup) = lookup {
1755 return lookup(endpoints);
1756 }
1757 endpoints
1758 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1759 .unwrap_or_default()
1760 })
1761 .await
1762 .unwrap_or_default()
1763 }
1764
1765 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1766 frame.header.channel == 0
1767 && frame.header.ty == FrameType::Request
1768 && self
1769 .forwarding
1770 .module_endpoint_for_connection(ctx.connection_id)
1771 .is_ok_and(|endpoint| endpoint.is_none())
1772 && parse_client_control_request(&frame.body)
1773 .ok()
1774 .is_some_and(|request| privileged_operation(&request).is_some())
1775 }
1776
1777 pub(crate) async fn handle_control_frame_timed(
1778 &self,
1779 ctx: &RouteCtx,
1780 frame: Frame,
1781 dispatch_started_at: Option<StdInstant>,
1782 ) -> Result<Vec<Frame>, RouterError> {
1783 match frame.header.ty {
1784 FrameType::Ping => Ok(vec![pong(&frame)?]),
1785 FrameType::Hello => {
1786 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1787 }
1788 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1789 FrameType::Cancel => {
1790 if self
1793 .forwarding
1794 .operator_confirms()
1795 .cancel(ctx.connection_id, frame.header.corr)
1796 || self
1797 .supervisor
1798 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1799 || self.cancel_activity_subscription(ctx, &frame)?
1800 {
1801 Ok(Vec::new())
1802 } else {
1803 Ok(vec![control_error_frame(
1804 &frame,
1805 "unknown_subscription",
1806 "no subscription has this correlation id",
1807 )?])
1808 }
1809 }
1810 FrameType::Request => {
1811 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1814 if op
1815 .as_ref()
1816 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1817 {
1818 return self.handle_activity_record(ctx, frame).await;
1819 }
1820 if op.as_ref().is_some_and(|probe| {
1821 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1822 }) {
1823 return self.handle_activity_read(ctx, frame).await;
1824 }
1825 if op
1826 .as_ref()
1827 .is_some_and(|probe| probe.op == "operator.confirm")
1828 {
1829 return self.handle_operator_confirm(ctx, frame);
1830 }
1831 if op
1832 .as_ref()
1833 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1834 {
1835 return self.handle_scope_served_by(ctx.connection_id, frame);
1836 }
1837 if self
1838 .forwarding
1839 .module_endpoint_for_connection(ctx.connection_id)
1840 .map_err(RouterError::Forwarding)?
1841 .is_some()
1842 {
1843 if !is_known_module_request_op(&frame.body) {
1844 return Ok(vec![control_error_frame(
1845 &frame,
1846 "unsupported_control_frame",
1847 "module-originated channel-0 REQUEST is not supported",
1848 )?]);
1849 }
1850 let request = match parse_module_control_request_from_module(&frame.body) {
1851 Ok(request) => request,
1852 Err((err, ControlRequestBodyError::UnknownOp)) => {
1853 return Ok(vec![control_error_frame(
1854 &frame,
1855 "unsupported_control_frame",
1856 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1857 )?])
1858 }
1859 Err((err, ControlRequestBodyError::InvalidBody)) => {
1860 return Ok(vec![control_error_frame(
1861 &frame,
1862 "invalid_control_body",
1863 format!("malformed module control body: {err}"),
1864 )?])
1865 }
1866 };
1867 let op = module_control_request_op(&request);
1868 let corr = frame.header.corr;
1869 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1870 let result =
1871 self.handle_module_control_request(ctx.connection_id, frame, request);
1872 log_slow_control_dispatch(
1873 dispatch_started_at,
1874 op,
1875 ctx.connection_id,
1876 corr,
1877 None,
1878 );
1879 return result;
1880 }
1881
1882 if is_known_module_request_op(&frame.body) {
1883 return Ok(vec![control_error_frame(
1884 &frame,
1885 "not_registered",
1886 "catalog.update requires an active module registration owned by this connection",
1887 )?]);
1888 }
1889
1890 let request = match parse_client_control_request(&frame.body) {
1891 Ok(request) => request,
1892 Err((err, ControlRequestBodyError::UnknownOp)) => {
1893 return Ok(vec![control_error_frame(
1894 &frame,
1895 "unknown_control_op",
1896 format!("unknown client control op: {err}"),
1897 )?])
1898 }
1899 Err((err, ControlRequestBodyError::InvalidBody)) => {
1900 return Ok(vec![control_error_frame(
1901 &frame,
1902 "invalid_control_body",
1903 format!("malformed client control body: {err}"),
1904 )?])
1905 }
1906 };
1907 let op = client_control_request_op(&request);
1908 let corr = frame.header.corr;
1909 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1910 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1911 #[cfg(test)]
1912 if let Some(delay) = self.control_dispatch_delay {
1913 tokio::time::sleep(delay).await;
1914 }
1915 let result = self
1916 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1917 .await;
1918 if let Some(timings) = route_open_timings.as_mut() {
1919 timings.finish();
1920 }
1921 log_slow_control_dispatch(
1922 dispatch_started_at,
1923 op,
1924 ctx.connection_id,
1925 corr,
1926 route_open_timings.as_ref(),
1927 );
1928 result
1929 }
1930 FrameType::Push => {
1931 let Some(endpoint) = self
1932 .forwarding
1933 .module_endpoint_for_connection(ctx.connection_id)
1934 .map_err(RouterError::Forwarding)?
1935 else {
1936 return Ok(vec![control_error_frame(
1937 &frame,
1938 "unsupported_control_frame",
1939 "client-originated channel-0 PUSH is not supported",
1940 )?]);
1941 };
1942 self.handle_status_update(endpoint, frame)
1943 }
1944 FrameType::Response | FrameType::Error
1945 if self
1946 .forwarding
1947 .module_endpoint_for_connection(ctx.connection_id)
1948 .map_err(RouterError::Forwarding)?
1949 .is_some() =>
1950 {
1951 self.handle_module_relay_response(ctx.connection_id, frame)
1952 }
1953 ty => Ok(vec![control_error_frame(
1954 &frame,
1955 "unsupported_control_frame",
1956 format!("unsupported channel-0 frame {ty:?}"),
1957 )?]),
1958 }
1959 }
1960
1961 pub fn cleanup_connection(
1962 &self,
1963 connection_id: ConnectionId,
1964 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1965 self.caller_endpoints
1966 .lock()
1967 .unwrap_or_else(|p| p.into_inner())
1968 .remove(&connection_id);
1969 self.forwarding
1970 .operator_confirms()
1971 .connection_closed(connection_id);
1972 self.cleanup_connection_with_end_reason(
1973 connection_id,
1974 RegistrationEndReason::ConnectionClosed,
1975 )
1976 }
1977
1978 fn cleanup_connection_with_end_reason(
1979 &self,
1980 connection_id: ConnectionId,
1981 requested_reason: RegistrationEndReason,
1982 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1983 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1984 self.registry
1985 .get_module_by_connection(connection_id)?
1986 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1987 .and_then(|module| module.registration_end_reason().ok().flatten())
1988 .unwrap_or(requested_reason)
1989 } else {
1990 requested_reason
1991 };
1992 let crash_closed = self
1993 .registry
1994 .get_module_by_connection(connection_id)?
1995 .and_then(|registration| {
1996 self.forwarding
1997 .module_endpoint_for_connection(connection_id)
1998 .ok()
1999 .flatten()
2000 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2001 .map(|routes| (registration.manifest.module_id, routes))
2002 });
2003 let crash_closed = crash_closed.map(|(module_id, routes)| {
2004 let terminal = match self.supervisor.get(&module_id) {
2005 None => false,
2006 Some(module) => match module.will_recover_after_connection_loss() {
2007 Ok(will_recover) => !will_recover,
2008 Err(err) => {
2009 warn!(
2010 %module_id,
2011 error = %err,
2012 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2013 );
2014 false
2015 }
2016 },
2017 };
2018 let (reason, drained) = match self
2022 .forwarding
2023 .daemon_drain_result_for_connection(connection_id)
2024 {
2025 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2026 Ok(None) => (RouteCloseReason::Crash, false),
2027 Err(err) => {
2028 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2029 (RouteCloseReason::Crash, false)
2030 }
2031 };
2032 (module_id, routes, reason, drained, terminal)
2033 });
2034 let registrations = self.deregister_connection(connection_id, end_reason);
2035 let cleanup = if crash_closed.is_some() {
2036 self.forwarding.cleanup_connection_counted(connection_id)
2037 } else {
2038 self.forwarding
2044 .cleanup_connection(connection_id)
2045 .map(|released| crate::forwarding::ConnectionCleanup {
2046 released,
2047 abandoned_relays: 0,
2048 })
2049 };
2050 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2055 let abandoned = cleanup
2056 .as_ref()
2057 .map_or(0, |cleanup| cleanup.abandoned_relays);
2058 send_route_control_pushes(
2059 &self.forwarding,
2060 routes,
2061 ClientControlPush::RouteClosed {
2062 module_id,
2063 channels: Vec::new(),
2064 reason,
2065 drained,
2066 abandoned,
2067 excluded_subscriptions: 0,
2068 terminal: Some(terminal),
2069 },
2070 );
2071 }
2072 if let Ok(cleanup) = cleanup {
2073 self.emit_route_goodbyes(cleanup.released);
2074 }
2075 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2079 crate::supervise::notify_registration_release();
2080 self.capability_evaluator.wake_deadline_loop();
2081 self.refresh_capability_requirements();
2082 }
2083 self.supervisor.remove_spawn_subscribers(connection_id);
2084 self.activity_streams
2085 .lock()
2086 .unwrap_or_else(|p| p.into_inner())
2087 .retain(|(id, _), task| {
2088 if *id == connection_id {
2089 task.abort();
2090 false
2091 } else {
2092 true
2093 }
2094 });
2095 self.hello_launch_nonces
2098 .lock()
2099 .unwrap_or_else(|poisoned| poisoned.into_inner())
2100 .forget(connection_id);
2101 self.scopes
2102 .write()
2103 .unwrap_or_else(|poisoned| poisoned.into_inner())
2104 .release_connection(connection_id);
2105 registrations
2106 }
2107
2108 pub(crate) fn handle_route_goodbye(
2109 &self,
2110 connection_id: ConnectionId,
2111 route_channel: u16,
2112 route_epoch: u32,
2113 ) -> Result<bool, RouterError> {
2114 debug!(
2115 connection_id = connection_id.get(),
2116 route_channel, route_epoch, "handling route GOODBYE"
2117 );
2118 let RouteRelease::Removed(released_route) = self
2119 .forwarding
2120 .release_client_route(connection_id, route_channel, route_epoch)
2121 .map_err(RouterError::Forwarding)?
2122 else {
2123 return Ok(false);
2124 };
2125 self.emit_route_goodbyes(vec![released_route]);
2126 Ok(true)
2127 }
2128
2129 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2130 for released in released_routes {
2131 let frame = match Frame::build_with_version(
2132 released.negotiated_ver,
2133 FrameType::Goodbye,
2134 control_flags(),
2135 released.channel,
2136 released.epoch,
2137 0,
2138 Vec::new(),
2139 ) {
2140 Ok(frame) => frame,
2141 Err(err) => {
2142 warn!(
2143 route_channel = released.channel,
2144 error = %err,
2145 "failed to build route GOODBYE frame"
2146 );
2147 continue;
2148 }
2149 };
2150 if !released.close_on_delivery_failure() {
2151 crate::forwarding::send_module_route_goodbye(
2152 &self.counters,
2153 &released.sink,
2154 frame,
2155 released.module_id.as_deref(),
2156 "client route released",
2157 );
2158 continue;
2159 }
2160 if let Err(err) = released.sink.try_send(frame) {
2161 warn!(
2162 target_connection_id = released.connection_id.get(),
2163 route_channel = released.channel,
2164 error = %err,
2165 "route GOODBYE was not delivered to client; closing target connection"
2166 );
2167 if self
2168 .forwarding
2169 .escalate_client_delivery_failure(
2170 released.connection_id,
2171 released.channel,
2172 released.epoch,
2173 CloseReason::new(
2174 "route_goodbye_delivery_failed",
2175 format!(
2176 "failed to enqueue route GOODBYE for channel {}: {err}",
2177 released.channel
2178 ),
2179 ),
2180 crate::forwarding::UndeliveredFrame {
2181 module_id: released.module_id.as_deref(),
2182 sink: &released.sink,
2183 },
2184 )
2185 .unwrap_or(false)
2186 {
2187 self.counters.increment_goodbye_relay_client_failed();
2188 }
2189 }
2190 }
2191 }
2192
2193 fn send_abandoned_route_bind_goodbye(
2205 &self,
2206 module_sink: &crate::FrameSink,
2207 negotiated_ver: u8,
2208 module_channel: u16,
2209 module_epoch: u32,
2210 ) {
2211 let frame = match Frame::build_with_version(
2212 negotiated_ver,
2213 FrameType::Goodbye,
2214 control_flags(),
2215 module_channel,
2216 module_epoch,
2217 0,
2218 Vec::new(),
2219 ) {
2220 Ok(frame) => frame,
2221 Err(err) => {
2222 warn!(
2223 route_channel = module_channel,
2224 error = %err,
2225 "failed to build GOODBYE for abandoned route.bind"
2226 );
2227 return;
2228 }
2229 };
2230 crate::forwarding::send_module_route_goodbye(
2231 &self.counters,
2232 module_sink,
2233 frame,
2234 None,
2235 "abandoned route.bind",
2236 );
2237 }
2238
2239 fn handle_hello(
2240 &self,
2241 connection_id: ConnectionId,
2242 sink: Option<crate::FrameSink>,
2243 frame: Frame,
2244 ) -> Result<Vec<Frame>, RouterError> {
2245 debug!(
2246 connection_id = connection_id.get(),
2247 corr = frame.header.corr,
2248 "handling HELLO"
2249 );
2250 if self
2254 .registry
2255 .get_module_by_connection(connection_id)
2256 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2257 .is_some()
2258 {
2259 return Ok(vec![control_error_frame(
2260 &frame,
2261 "invalid_hello",
2262 "connection is already registered as a module",
2263 )?]);
2264 }
2265 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2266 Ok(value) => value,
2267 Err(err) => {
2268 return Ok(vec![control_error_frame(
2269 &frame,
2270 "invalid_hello",
2271 format!("malformed HELLO body: {err}"),
2272 )?])
2273 }
2274 };
2275 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2276 return Ok(vec![control_error_frame(
2277 &frame,
2278 "invalid_capability_grammar",
2279 err.to_string(),
2280 )?]);
2281 }
2282 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2283 return Ok(vec![control_error_frame(
2284 &frame,
2285 "invalid_manifest",
2286 err.to_string(),
2287 )?]);
2288 }
2289 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2290 return Ok(vec![control_error_frame(
2291 &frame,
2292 "invalid_event_declaration",
2293 err.to_string(),
2294 )?]);
2295 }
2296 if let Some(provenance) = hello_value
2297 .get("manifest")
2298 .and_then(|manifest| manifest.get("provenance"))
2299 {
2300 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2301 return Ok(vec![control_error_frame(
2302 &frame,
2303 "invalid_manifest",
2304 format!("malformed manifest provenance: {err}"),
2305 )?]);
2306 }
2307 }
2308 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2309 Ok(hello) => hello,
2310 Err(err) => {
2311 return Ok(vec![control_error_frame(
2312 &frame,
2313 "invalid_hello",
2314 format!("malformed HELLO body: {err}"),
2315 )?])
2316 }
2317 };
2318
2319 if hello.protocol_ver != hello.manifest.protocol_ver {
2320 return Ok(vec![control_error_frame(
2321 &frame,
2322 "invalid_manifest",
2323 format!(
2324 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2325 hello.protocol_ver, hello.manifest.protocol_ver
2326 ),
2327 )?]);
2328 }
2329
2330 if hello.manifest.module_id.trim().is_empty() {
2331 return Ok(vec![control_error_frame(
2332 &frame,
2333 "invalid_manifest",
2334 "manifest module_id must not be empty",
2335 )?]);
2336 }
2337
2338 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2339 Ok(negotiated_ver) => negotiated_ver,
2340 Err(message) => {
2341 return Ok(vec![control_error_frame(
2342 &frame,
2343 "version_unsupported",
2344 message,
2345 )?])
2346 }
2347 };
2348
2349 let swap_admission = self
2357 .supervisor
2358 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2359 if swap_admission == SwapHelloAdmission::Refused {
2360 warn!(
2361 module_id = %hello.manifest.module_id,
2362 connection_id = connection_id.get(),
2363 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2364 );
2365 return Ok(vec![control_error_frame(
2366 &frame,
2367 "swap_token_invalid",
2368 format!(
2369 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2370 hello.manifest.module_id
2371 ),
2372 )?]);
2373 }
2374 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2375
2376 if let Some(rejection) = (!swap_candidate)
2384 .then(|| {
2385 self.supervisor.reserved_hello_rejection(
2386 &hello.manifest.module_id,
2387 hello.launch_nonce.as_deref(),
2388 )
2389 })
2390 .flatten()
2391 {
2392 let message = match rejection {
2393 ReservedHelloRejection::Exact { module_id } => format!(
2394 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2395 ),
2396 ReservedHelloRejection::Prefix {
2397 prefix,
2398 owner_module_id,
2399 } => format!(
2400 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2401 hello.manifest.module_id
2402 ),
2403 };
2404 return Ok(vec![control_error_frame(
2405 &frame,
2406 "reserved_module",
2407 message,
2408 )?]);
2409 }
2410
2411 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2412 &hello.manifest.module_id,
2413 hello.manifest.capabilities.as_ref(),
2414 );
2415 if let Some(refusal) = reserved_capability_refusals.first() {
2416 let capability = refusal.capability.clone();
2417 let bound_module = refusal.claimants[0].clone();
2418 log_duplicate_claim_events(reserved_capability_refusals);
2419 return Ok(vec![control_error_frame(
2420 &frame,
2421 "reserved_capability",
2422 format!(
2423 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2424 capability, bound_module, hello.manifest.module_id
2425 ),
2426 )?]);
2427 }
2428
2429 if self
2432 .forwarding
2433 .connection_has_client_routes(connection_id)
2434 .map_err(RouterError::Forwarding)?
2435 {
2436 return Ok(vec![control_error_frame(
2437 &frame,
2438 "invalid_hello",
2439 "connection has open client routes and cannot also register as a module",
2440 )?]);
2441 }
2442
2443 self.hello_launch_nonces
2448 .lock()
2449 .unwrap_or_else(|poisoned| poisoned.into_inner())
2450 .record(connection_id, hello.launch_nonce.as_deref());
2451 let control_ops = effective_module_control_ops(hello.control_ops);
2452 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2455 if swap_candidate {
2456 return self.register_swap_candidate(
2457 connection_id,
2458 sink,
2459 &frame,
2460 hello.manifest,
2461 negotiated_ver,
2462 control_ops,
2463 hello_ack,
2464 );
2465 }
2466 let registration = match self.registry.register_unpublished_with_control_ops(
2473 hello.manifest,
2474 negotiated_ver,
2475 connection_id,
2476 control_ops,
2477 ) {
2478 Ok(registration) => registration,
2479 Err(RegistryError::DuplicateModuleId { module_id }) => {
2480 return Ok(vec![control_error_frame(
2481 &frame,
2482 "duplicate_module_id",
2483 format!(
2484 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2485 ),
2486 )?])
2487 }
2488 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2489 return Ok(vec![control_error_frame(
2490 &frame,
2491 "invalid_module_id",
2492 err.to_string(),
2493 )?])
2494 }
2495 Err(err) => {
2496 return Ok(vec![control_error_frame(
2497 &frame,
2498 "registry_error",
2499 err.to_string(),
2500 )?])
2501 }
2502 };
2503 #[cfg(test)]
2504 self.hold_between_registry_and_forwarding();
2505
2506 let reply = if let Some(sink) = sink {
2507 let concurrency = manifest_concurrency(®istration.manifest);
2523 if let Err(err) = self.forwarding.register_module_connection_acked(
2524 connection_id,
2525 registration.manifest.module_id.clone(),
2526 negotiated_ver,
2527 concurrency,
2528 sink,
2529 hello_ack,
2530 ) {
2531 if matches!(
2536 self.deregister_connection(
2537 connection_id,
2538 RegistrationEndReason::RegistrationFailed,
2539 ),
2540 Ok(r) if !r.is_empty()
2541 ) {
2542 crate::supervise::notify_registration_release();
2543 }
2544 return Ok(vec![control_error_frame(
2545 &frame,
2546 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2547 "invalid_hello"
2548 } else {
2549 forwarding_error_code(&err)
2550 },
2551 err.to_string(),
2552 )?]);
2553 }
2554 Vec::new()
2555 } else {
2556 vec![hello_ack]
2559 };
2560
2561 if !self
2566 .registry
2567 .publish_registration(connection_id)
2568 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2569 {
2570 debug!(
2571 module_id = %registration.manifest.module_id,
2572 connection_id = connection_id.get(),
2573 "registration was removed before it could be published"
2574 );
2575 }
2576
2577 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2586 info!(
2587 module_id = %registration.manifest.module_id,
2588 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2589 );
2590 }
2591
2592 self.apply_registration_capabilities(®istration);
2593
2594 self.supervisor
2595 .record_activity_registration(®istration.manifest.module_id);
2596
2597 info!(
2598 module_id = %registration.manifest.module_id,
2599 module_version = %registration.manifest.module_version,
2600 negotiated_ver,
2601 routable_provider = manifest_provides_routable_role(®istration.manifest),
2602 connection_id = connection_id.get(),
2603 "module registered"
2604 );
2605
2606 Ok(reply)
2607 }
2608
2609 #[allow(clippy::too_many_arguments)]
2626 fn register_swap_candidate(
2627 &self,
2628 connection_id: ConnectionId,
2629 sink: Option<crate::FrameSink>,
2630 frame: &Frame,
2631 manifest: ModuleManifest,
2632 negotiated_ver: u8,
2633 control_ops: Vec<String>,
2634 hello_ack: Frame,
2635 ) -> Result<Vec<Frame>, RouterError> {
2636 let module_id = manifest.module_id.clone();
2637 let registration = match self.registry.register_unpublished_candidate_with_control_ops(
2638 manifest,
2639 negotiated_ver,
2640 connection_id,
2641 control_ops,
2642 ) {
2643 Ok(registration) => registration,
2644 Err(RegistryError::DuplicateModuleId { module_id }) => {
2645 return Ok(vec![control_error_frame(
2646 frame,
2647 "duplicate_module_id",
2648 format!(
2649 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2650 ),
2651 )?])
2652 }
2653 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2654 return Ok(vec![control_error_frame(
2655 frame,
2656 "invalid_module_id",
2657 err.to_string(),
2658 )?])
2659 }
2660 Err(err) => {
2661 return Ok(vec![control_error_frame(
2662 frame,
2663 "registry_error",
2664 err.to_string(),
2665 )?])
2666 }
2667 };
2668 #[cfg(test)]
2669 self.hold_between_registry_and_forwarding();
2670 let reply = if let Some(sink) = sink {
2671 let concurrency = manifest_concurrency(®istration.manifest);
2675 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2676 connection_id,
2677 module_id.clone(),
2678 negotiated_ver,
2679 concurrency,
2680 sink,
2681 hello_ack,
2682 ) {
2683 if matches!(
2684 self.deregister_connection(
2685 connection_id,
2686 RegistrationEndReason::RegistrationFailed,
2687 ),
2688 Ok(r) if !r.is_empty()
2689 ) {
2690 crate::supervise::notify_registration_release();
2691 }
2692 return Ok(vec![control_error_frame(
2693 frame,
2694 forwarding_error_code(&err),
2695 err.to_string(),
2696 )?]);
2697 }
2698 Vec::new()
2699 } else {
2700 vec![hello_ack]
2701 };
2702 if !self
2703 .registry
2704 .publish_registration(connection_id)
2705 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2706 {
2707 debug!(
2708 module_id = %module_id,
2709 connection_id = connection_id.get(),
2710 "swap candidate registration was removed before it could be published"
2711 );
2712 }
2713 self.supervisor.mark_swap_candidate_admitted(&module_id);
2714 info!(
2715 module_id = %module_id,
2716 module_version = %registration.manifest.module_version,
2717 negotiated_ver,
2718 ready = registration.ready,
2719 connection_id = connection_id.get(),
2720 "swap candidate registered; not routable until cutover"
2721 );
2722 Ok(reply)
2723 }
2724
2725 fn build_hello_ack(
2726 &self,
2727 frame: &Frame,
2728 negotiated_ver: u8,
2729 module_id: &str,
2730 ) -> Result<Frame, RouterError> {
2731 let ack = ModuleHelloAckBody {
2732 negotiated_ver,
2733 subc_ops: module_subc_ops(),
2734 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2735 storage: self
2736 .storage_config
2737 .as_ref()
2738 .map(|cfg| cfg.descriptor_for(module_id)),
2739 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2740 };
2741 let body = serde_json::to_vec(&ack).map_err(|err| {
2742 RouterError::backend(
2743 0,
2744 frame.header.corr,
2745 format!("failed to encode HELLO_ACK: {err}"),
2746 )
2747 })?;
2748
2749 Frame::build_with_version(
2750 negotiated_ver,
2751 FrameType::HelloAck,
2752 control_flags(),
2753 0,
2754 0,
2755 frame.header.corr,
2756 body,
2757 )
2758 .map_err(RouterError::FrameBuild)
2759 }
2760
2761 async fn handle_client_control_request(
2762 &self,
2763 ctx: &RouteCtx,
2764 frame: Frame,
2765 request: ClientControlRequest,
2766 route_open_timings: &mut Option<RouteOpenTimings>,
2767 ) -> Result<Vec<Frame>, RouterError> {
2768 if let Some(operation) = privileged_operation(&request) {
2769 let caller = self.resolve_caller(ctx.connection_id).await;
2770 let op = client_control_request_op(&request);
2771 let text = prompt_text(operation, &caller);
2772 match self.privileged_operator_prompt {
2773 PrivilegedOperatorPrompt::Log => {
2774 info!(target: "subc_daemon::privileged_prompt", operation = op,
2775 target = ?operation, caller = %caller.description(), summary = %text,
2776 "privileged request would prompt the operator");
2777 }
2778 PrivilegedOperatorPrompt::Enforce => {
2779 let key =
2780 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2781 let outcome = self
2782 .forwarding
2783 .operator_confirms()
2784 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2785 .await;
2786 if outcome.code != "confirmed" {
2787 return Ok(vec![outcome.frame(&frame)]);
2788 }
2789 }
2790 }
2791 }
2792 match request {
2793 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2794 ClientControlRequest::CatalogList { module_id } => {
2795 self.handle_catalog_list(frame, module_id)
2796 }
2797 ClientControlRequest::RouteOpen {
2798 target,
2799 identity,
2800 consumer_identity,
2801 consumer_capabilities,
2802 role_versions,
2803 admission_facts,
2804 scope,
2805 } => {
2806 self.handle_route_open(
2807 ctx,
2808 frame,
2809 RouteOpenRequest {
2810 target,
2811 identity,
2812 consumer_identity,
2813 consumer_capabilities,
2814 role_versions,
2815 admission_facts,
2816 scope,
2817 },
2818 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2819 )
2820 .await
2821 }
2822 ClientControlRequest::RoutePoll {
2823 route_channel,
2824 route_epoch,
2825 kind,
2826 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2827 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2828 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2829 self.handle_supervisor_spawn_snapshot(frame)
2830 }
2831 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2832 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2833 }
2834 ClientControlRequest::SupervisorRestart {
2835 module_id,
2836 drain_timeout_ms,
2837 } => {
2838 self.log_supervisor_request_received(
2839 ctx,
2840 frame.header.corr,
2841 ops::SUPERVISOR_RESTART,
2842 Some(&module_id),
2843 None,
2844 )?;
2845 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2846 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2847 .await
2848 }
2849 ClientControlRequest::SupervisorSwap {
2850 module_id,
2851 ready_timeout_ms,
2852 } => {
2853 self.log_supervisor_request_received(
2854 ctx,
2855 frame.header.corr,
2856 ops::SUPERVISOR_SWAP,
2857 Some(&module_id),
2858 None,
2859 )?;
2860 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2861 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2862 .await
2863 }
2864 ClientControlRequest::SupervisorReload { module_id } => {
2865 self.log_supervisor_request_received(
2866 ctx,
2867 frame.header.corr,
2868 ops::SUPERVISOR_RELOAD,
2869 Some(&module_id),
2870 None,
2871 )?;
2872 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2873 self.handle_supervisor_reload(frame, module_id, requester)
2874 .await
2875 }
2876 ClientControlRequest::SupervisorRescan { preview } => {
2877 if !preview {
2878 self.log_supervisor_request_received(
2879 ctx,
2880 frame.header.corr,
2881 ops::SUPERVISOR_RESCAN,
2882 None,
2883 None,
2884 )?;
2885 }
2886 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2887 self.handle_supervisor_rescan(frame, preview, requester)
2888 .await
2889 }
2890 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2891 self.log_supervisor_request_received(
2892 ctx,
2893 frame.header.corr,
2894 ops::SUPERVISOR_RELEASE_RESERVED,
2895 Some(&module_id),
2896 None,
2897 )?;
2898 self.handle_supervisor_release_reserved(frame, module_id)
2899 .await
2900 }
2901 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2902 self.log_supervisor_request_received(
2903 ctx,
2904 frame.header.corr,
2905 ops::SUPERVISOR_SET_ENABLED,
2906 Some(&module_id),
2907 Some(enabled),
2908 )?;
2909 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2910 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2911 .await
2912 }
2913 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2914 self.handle_supervisor_health_probe(frame, module_id).await
2915 }
2916 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2917 ClientControlRequest::SupervisorRoutes { module_id } => {
2918 self.handle_supervisor_routes(frame, module_id)
2919 }
2920 ClientControlRequest::SupervisorProvenance { module_id } => {
2921 self.handle_supervisor_provenance(frame, module_id).await
2922 }
2923 ClientControlRequest::SupervisorStderrTail {
2924 module_id,
2925 max_lines,
2926 max_bytes,
2927 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2928 ClientControlRequest::SupervisorTerminals { module_id } => {
2929 self.handle_supervisor_terminals(frame, module_id).await
2930 }
2931 }
2932 }
2933
2934 fn log_supervisor_request_received(
2935 &self,
2936 ctx: &RouteCtx,
2937 corr: u64,
2938 op: &'static str,
2939 module_id: Option<&str>,
2940 enabled: Option<bool>,
2941 ) -> Result<(), RouterError> {
2942 let caller = self.supervisor_requester(ctx, corr)?;
2943 let caller = principal_label(&caller);
2944 info!(
2947 target: "control",
2948 op,
2949 module_id,
2950 enabled,
2951 connection_id = ctx.connection_id.get(),
2952 caller = %caller,
2953 "supervisor request received"
2954 );
2955 Ok(())
2956 }
2957
2958 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2959 Ok(self
2960 .registry
2961 .get_module_by_connection(ctx.connection_id)
2962 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2963 .map(|registration| Principal::Reserved {
2964 module_id: registration.manifest.module_id,
2965 })
2966 .unwrap_or(Principal::Direct))
2967 }
2968
2969 fn handle_module_control_request(
2970 &self,
2971 connection_id: ConnectionId,
2972 frame: Frame,
2973 request: ModuleControlRequestFromModule,
2974 ) -> Result<Vec<Frame>, RouterError> {
2975 match request {
2976 ModuleControlRequestFromModule::CatalogUpdate {
2977 provides,
2978 capabilities,
2979 ready,
2980 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2981 ModuleControlRequestFromModule::LiveRoots {} => {
2982 let registered = self
2983 .registry
2984 .get_module_by_connection(connection_id)
2985 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2986 let Some(registration) = registered else {
2987 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2988 };
2989 let response = self
2990 .forwarding
2991 .live_roots(®istration.manifest.module_id)
2992 .map_err(RouterError::Forwarding)?;
2993 Ok(vec![control_response_body_frame(
2994 &frame,
2995 &response,
2996 "ModuleControlResponseToModule::LiveRoots",
2997 )?])
2998 }
2999 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3000 self.handle_scope_sync(connection_id, frame, generation, scopes)
3001 }
3002 ModuleControlRequestFromModule::ScopeApply {
3003 generation,
3004 upsert,
3005 end,
3006 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3007 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3008 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3009 }
3010 }
3011 }
3012
3013 fn handle_scope_served_by(
3014 &self,
3015 connection_id: ConnectionId,
3016 frame: Frame,
3017 ) -> Result<Vec<Frame>, RouterError> {
3018 let registered = self
3019 .registry
3020 .get_module_by_connection(connection_id)
3021 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3022 let Some(registered) = registered else {
3023 return Ok(vec![control_error_frame(
3024 &frame,
3025 "not_registered",
3026 "scope.served_by requires an active module registration owned by this connection",
3027 )?]);
3028 };
3029 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3030 {
3031 Ok(request) => request,
3032 Err(err) => {
3033 return Ok(vec![control_error_frame(
3034 &frame,
3035 "invalid_control_body",
3036 format!("malformed scope.served_by body: {err}"),
3037 )?])
3038 }
3039 };
3040 let answer = self
3041 .scopes
3042 .read()
3043 .unwrap_or_else(|poisoned| poisoned.into_inner())
3044 .served_by(
3045 ®istered.manifest.module_id,
3046 &request.scope_ref,
3047 request.scope_epoch,
3048 &request.module_id,
3049 );
3050 let reply = subc_control::ScopeServedByReply::new(
3051 answer,
3052 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3053 );
3054 Ok(vec![control_response_body_frame(
3055 &frame,
3056 &reply,
3057 "ScopeServedByReply",
3058 )?])
3059 }
3060
3061 fn handle_operator_confirm(
3062 &self,
3063 ctx: &RouteCtx,
3064 frame: Frame,
3065 ) -> Result<Vec<Frame>, RouterError> {
3066 use crate::operator_confirm::{audit, Outcome};
3067 let registration = self
3068 .registry
3069 .get_module_by_connection(ctx.connection_id)
3070 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3071 let Some(registration) = registration else {
3072 let outcome = Outcome::refusal("not_registered");
3073 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3074 return Ok(vec![outcome.frame(&frame)]);
3075 };
3076 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3077 Ok(request) => request,
3078 Err(_) => {
3079 let outcome = Outcome::refusal("invalid_control_body");
3080 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3081 return Ok(vec![outcome.frame(&frame)]);
3082 }
3083 };
3084 let module_id = registration.manifest.module_id;
3085 let nonce = self
3089 .hello_launch_nonces
3090 .lock()
3091 .unwrap_or_else(|p| p.into_inner())
3092 .nonce(ctx.connection_id)
3093 .map(str::to_owned);
3094 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3095 self.supervisor
3096 .spawned_consumer_authorized(&module_id, nonce)
3097 });
3098 let confirms = self.forwarding.operator_confirms();
3099 self.forwarding
3100 .with_operator_route(
3101 ctx.connection_id,
3102 request.route_channel,
3103 request.route_epoch,
3104 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3105 )
3106 .map_err(RouterError::Forwarding)
3107 }
3108
3109 fn handle_scope_sync(
3113 &self,
3114 connection_id: ConnectionId,
3115 frame: Frame,
3116 generation: u64,
3117 scopes: Vec<ScopeRecord>,
3118 ) -> Result<Vec<Frame>, RouterError> {
3119 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3120 }
3121
3122 fn handle_scope_change(
3123 &self,
3124 connection_id: ConnectionId,
3125 frame: Frame,
3126 generation: u64,
3127 scopes: Vec<ScopeRecord>,
3128 end: Option<Vec<ScopeEnd>>,
3129 ) -> Result<Vec<Frame>, RouterError> {
3130 let is_apply = end.is_some();
3131 let Some(registration) = self
3132 .registry
3133 .get_module_by_connection(connection_id)
3134 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3135 else {
3136 return Ok(vec![control_error_frame(
3137 &frame,
3138 "not_registered",
3139 "scope.sync and scope.apply require an active module registration owned by this connection",
3140 )?]);
3141 };
3142 let owner = registration.manifest.module_id;
3143 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3144 let is_current_launch = |connection: ConnectionId| {
3145 self.hello_launch_nonces
3146 .lock()
3147 .unwrap_or_else(|poisoned| poisoned.into_inner())
3148 .presented(connection, current_nonce.as_deref())
3149 };
3150 let mut table = self
3155 .scopes
3156 .write()
3157 .unwrap_or_else(|poisoned| poisoned.into_inner());
3158 let outcome = match end {
3159 Some(end) => table.apply(
3160 &owner,
3161 connection_id,
3162 is_current_launch,
3163 generation,
3164 scopes,
3165 end,
3166 ),
3167 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3168 };
3169 let drained = match &outcome {
3170 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3171 Err(_) => Vec::new(),
3172 };
3173 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3174 drop(table);
3175 if has_deadlines {
3176 self.scope_deadline_added.notify_one();
3177 }
3178 match outcome {
3179 Ok(applied) => {
3180 let counts = ScopeOutcomeCounts::of(&applied.results);
3181 info!(
3182 owner = %owner,
3183 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3184 generation,
3185 records = applied.results.len(),
3186 created = counts.created,
3187 replaced = counts.replaced,
3188 updated = counts.updated,
3189 unchanged = counts.unchanged,
3190 refused = counts.refused,
3191 ended = applied.ended.len(),
3192 tag_changes = applied.tag_changes.len(),
3193 routes_closed = drained.len(),
3194 "scope change accepted"
3195 );
3196 for refused in applied
3202 .results
3203 .iter()
3204 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3205 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3206 {
3207 warn!(
3208 owner = %owner,
3209 generation,
3210 scope_ref = %refused.scope_ref,
3211 scope_epoch = refused.scope_epoch,
3212 code = refused.code.as_deref().unwrap_or(""),
3213 "scope record refused"
3214 );
3215 }
3216 self.close_scope_drained_routes(drained);
3217 let response = if is_apply {
3218 ModuleControlResponseToModule::ScopeApply {
3219 generation,
3220 results: applied.results,
3221 end_results: applied.end_results,
3222 ended: applied.ended,
3223 }
3224 } else {
3225 ModuleControlResponseToModule::ScopeSync {
3226 generation,
3227 results: applied.results,
3228 ended: applied.ended,
3229 }
3230 };
3231 Ok(vec![control_response_body_frame(
3232 &frame,
3233 &response,
3234 "scope change response",
3235 )?])
3236 }
3237 Err(refusal) => {
3238 info!(
3239 owner = %owner,
3240 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3241 generation,
3242 code = refusal.code,
3243 "scope change refused"
3244 );
3245 Ok(vec![control_error_frame(
3246 &frame,
3247 refusal.code,
3248 refusal.message,
3249 )?])
3250 }
3251 }
3252 }
3253
3254 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3260 if drained.is_empty() {
3261 return;
3262 }
3263 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3264 BTreeMap::new();
3265 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3266 for route in drained {
3267 warn!(
3268 module_id = %route.module_id,
3269 reason = ?route.reason,
3270 client_connection_id = route.client.connection_id.get(),
3271 route_channel = route.client.channel,
3272 "closing route because its scope changed"
3273 );
3274 pushes
3275 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3276 .or_insert_with(|| (route.reason, Vec::new()))
3277 .1
3278 .push(EndpointRoute {
3279 goodbye_target: route.client.clone(),
3280 principal: Principal::Unverified,
3281 bound_at: Instant::now(),
3282 draining: false,
3283 drain_reason: None,
3284 });
3285 goodbyes.push(route.module);
3286 goodbyes.push(route.client);
3287 }
3288 for ((module_id, _), (reason, routes)) in pushes {
3289 send_route_control_pushes(
3290 &self.forwarding,
3291 routes,
3292 ClientControlPush::RouteClosed {
3293 module_id,
3294 channels: Vec::new(),
3295 reason,
3296 drained: false,
3297 abandoned: 0,
3298 excluded_subscriptions: 0,
3299 terminal: Some(false),
3300 },
3301 );
3302 }
3303 self.emit_route_goodbyes(goodbyes);
3304 }
3305
3306 fn handle_scope_describe(
3309 &self,
3310 connection_id: ConnectionId,
3311 frame: Frame,
3312 owner: Principal,
3313 scope_ref: String,
3314 ) -> Result<Vec<Frame>, RouterError> {
3315 let registered = self
3316 .registry
3317 .get_module_by_connection(connection_id)
3318 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3319 let Some(registered) = registered else {
3320 return Ok(vec![control_error_frame(
3321 &frame,
3322 "not_registered",
3323 "scope.describe requires an active module registration owned by this connection",
3324 )?]);
3325 };
3326 let mut description = self
3327 .scopes
3328 .read()
3329 .unwrap_or_else(|poisoned| poisoned.into_inner())
3330 .describe(&owner, &scope_ref);
3331 if !registered
3334 .manifest
3335 .capabilities
3336 .as_ref()
3337 .is_some_and(|caps| {
3338 caps.provides
3339 .iter()
3340 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3341 })
3342 {
3343 if let Some(stamp) = description.stamp.as_mut() {
3344 stamp.attributes.on_behalf_of_agent = None;
3345 }
3346 }
3347 let owner_configured = match &owner {
3348 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3353 _ => false,
3354 };
3355 let response = ModuleControlResponseToModule::ScopeDescribe {
3356 status: description.status,
3357 scope_epoch: description.scope_epoch,
3358 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3359 owner_synced: description.owner_synced,
3360 owner_configured,
3361 scope: description.stamp,
3362 };
3363 Ok(vec![control_response_body_frame(
3364 &frame,
3365 &response,
3366 "ModuleControlResponseToModule::ScopeDescribe",
3367 )?])
3368 }
3369
3370 fn handle_catalog_update(
3371 &self,
3372 connection_id: ConnectionId,
3373 frame: Frame,
3374 provides: Vec<ProviderRole>,
3375 capabilities: Option<CapabilityDeclarations>,
3376 ready: Option<bool>,
3377 ) -> Result<Vec<Frame>, RouterError> {
3378 self.refresh_capability_requirements();
3379 let Some(registration) = self
3380 .registry
3381 .get_module_by_connection(connection_id)
3382 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3383 else {
3384 return Ok(vec![control_error_frame(
3385 &frame,
3386 "not_registered",
3387 "catalog.update requires an active module registration owned by this connection",
3388 )?]);
3389 };
3390
3391 if let Some(message) =
3392 catalog_update_frozen_field_message(®istration.manifest, &provides)
3393 {
3394 return Ok(vec![control_error_frame(
3395 &frame,
3396 "catalog_update_frozen_field",
3397 message,
3398 )?]);
3399 }
3400
3401 let mut candidate = registration.manifest.clone();
3402 candidate.provides = provides.clone();
3403 candidate.capabilities = capabilities
3404 .clone()
3405 .or_else(|| registration.manifest.capabilities.clone());
3406 if let Err(err) = candidate.validate_capability_grammar() {
3407 return Ok(vec![control_error_frame(
3408 &frame,
3409 "invalid_capability_grammar",
3410 err.to_string(),
3411 )?]);
3412 }
3413
3414 let mut conflicts = self
3417 .capability_evaluator
3418 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3419 if let Some(conflict) = conflicts.first() {
3420 let message = format!(
3421 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3422 conflict.capability, conflict.claimants[0], candidate.module_id
3423 );
3424 for conflict in &mut conflicts {
3425 conflict.source = DuplicateClaimSource::CatalogUpdate;
3426 }
3427 log_duplicate_claim_events(conflicts);
3428 return Ok(vec![control_error_frame(
3429 &frame,
3430 "reserved_capability",
3431 message,
3432 )?]);
3433 }
3434
3435 let updated = self
3436 .registry
3437 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3438 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3439 if updated.is_none() {
3440 return Ok(vec![control_error_frame(
3441 &frame,
3442 "not_registered",
3443 "catalog.update requires an active module registration owned by this connection",
3444 )?]);
3445 }
3446 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3447 log_duplicate_claim_events(
3448 self.capability_evaluator
3449 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3450 );
3451 }
3452 if capability_census_trigger(
3453 registration.manifest.capabilities.as_ref(),
3454 updated
3455 .as_ref()
3456 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3457 ) {
3458 self.enforce_capability_denies();
3459 }
3460 self.refresh_capability_requirements();
3461
3462 let response = ModuleControlResponseToModule::CatalogUpdate {};
3463 control_response_body_frame(
3464 &frame,
3465 &response,
3466 "ModuleControlResponseToModule::CatalogUpdate",
3467 )
3468 .map(|frame| vec![frame])
3469 }
3470
3471 pub(crate) fn with_activity_ledger_config(
3472 mut self,
3473 run_dir: &Path,
3474 incarnation: String,
3475 config: crate::activity::ActivityConfig,
3476 ) -> Self {
3477 self.activity = Some(crate::activity::ActivityLedger::open_config(
3478 run_dir.join("activity.jsonl"),
3479 incarnation,
3480 config,
3481 ));
3482 self.supervisor.set_activity(self.activity.clone());
3483 self
3484 }
3485
3486 pub(crate) async fn configure_activity(
3487 &self,
3488 config: crate::activity::ActivityConfig,
3489 ) -> Result<(), &'static str> {
3490 match &self.activity {
3491 Some(ledger) => ledger.configure(config).await,
3492 None => Ok(()),
3493 }
3494 }
3495
3496 fn cancel_activity_subscription(
3497 &self,
3498 ctx: &RouteCtx,
3499 frame: &Frame,
3500 ) -> Result<bool, RouterError> {
3501 let task = self
3502 .activity_streams
3503 .lock()
3504 .unwrap_or_else(|p| p.into_inner())
3505 .remove(&(ctx.connection_id, frame.header.corr));
3506 let Some(task) = task else {
3507 return Ok(false);
3508 };
3509 task.abort();
3510 let end = Frame::build_with_version(
3511 response_version(frame),
3512 FrameType::StreamEnd,
3513 control_flags(),
3514 0,
3515 0,
3516 frame.header.corr,
3517 Vec::new(),
3518 )
3519 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3520 let sink = ctx.egress.clone();
3521 tokio::spawn(async move {
3522 let _ = sink.send(end).await;
3523 });
3524 Ok(true)
3525 }
3526
3527 async fn handle_activity_read(
3528 &self,
3529 ctx: &RouteCtx,
3530 frame: Frame,
3531 ) -> Result<Vec<Frame>, RouterError> {
3532 use subc_control::{
3533 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3534 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3535 };
3536 #[derive(Deserialize)]
3537 struct Identity {
3538 #[serde(default)]
3539 consumer_identity: Option<ConsumerIdentity>,
3540 }
3541 let principal = match self
3545 .forwarding
3546 .module_id_for_connection(ctx.connection_id)
3547 .map_err(RouterError::Forwarding)?
3548 {
3549 Some(module_id) => Ok(Principal::Reserved { module_id }),
3550 None => {
3551 let identity: Identity = match serde_json::from_slice(&frame.body) {
3552 Ok(identity) => identity,
3553 Err(_) => {
3554 return Ok(vec![control_error_frame(
3555 &frame,
3556 ACTIVITY_MALFORMED,
3557 "malformed activity read",
3558 )?])
3559 }
3560 };
3561 self.route_open_principal(&frame, identity.consumer_identity)?
3562 }
3563 };
3564 let admitted = matches!(principal, Ok(Principal::Direct))
3565 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3566 if !admitted {
3567 return Ok(vec![control_error_frame(
3568 &frame,
3569 ACTIVITY_READ_FORBIDDEN,
3570 "activity reads require an operator or trusted mirror principal",
3571 )?]);
3572 }
3573 let Some(ledger) = &self.activity else {
3574 return Ok(vec![control_error_frame(
3575 &frame,
3576 ACTIVITY_UNAVAILABLE,
3577 "activity ledger unavailable",
3578 )?]);
3579 };
3580 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3581 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3582 {
3583 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3584 Ok(request) => ledger.query(request).await,
3585 Err(_) => Err(ACTIVITY_MALFORMED),
3586 };
3587 return match result {
3588 Ok(reply) => Ok(vec![control_response_body_frame(
3589 &frame,
3590 &reply,
3591 ops::ACTIVITY_QUERY,
3592 )?]),
3593 Err(code) => Ok(vec![control_error_frame(
3594 &frame,
3595 code,
3596 "activity query refused",
3597 )?]),
3598 };
3599 }
3600 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3601 Ok(request) => request,
3602 Err(_) => {
3603 return Ok(vec![control_error_frame(
3604 &frame,
3605 ACTIVITY_MALFORMED,
3606 "malformed activity subscription",
3607 )?])
3608 }
3609 };
3610 let mut subscription = match ledger.subscribe(request.since).await {
3611 Ok(subscription) => subscription,
3612 Err(code) => {
3613 return Ok(vec![control_error_frame(
3614 &frame,
3615 code,
3616 "activity subscription refused",
3617 )?])
3618 }
3619 };
3620 let key = (ctx.connection_id, frame.header.corr);
3621 let sink = ctx.egress.clone();
3622 let mut streams = self
3623 .activity_streams
3624 .lock()
3625 .unwrap_or_else(|p| p.into_inner());
3626 streams.retain(|_, task| !task.is_finished());
3627 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3628 return Ok(vec![control_error_frame(
3629 &frame,
3630 "duplicate_subscription",
3631 "activity subscription correlation id already in use",
3632 )?]);
3633 }
3634 let task = tokio::spawn(async move {
3635 loop {
3636 let entry = if let Some(entry) = subscription.replay.pop_front() {
3637 Some(entry)
3638 } else {
3639 subscription.receiver.recv().await
3640 };
3641 let Some(entry) = entry else {
3642 break;
3643 };
3644 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3645 return;
3646 };
3647 let Ok(next) = Frame::build_with_version(
3648 response_version(&frame),
3649 FrameType::StreamData,
3650 control_flags(),
3651 0,
3652 0,
3653 frame.header.corr,
3654 body,
3655 ) else {
3656 return;
3657 };
3658 if sink.send(next).await.is_err() {
3659 return;
3660 }
3661 }
3662 if let Some(code) = subscription.terminal_code() {
3665 if let Ok(end) =
3666 control_error_frame(&frame, code, "activity subscriber fell behind")
3667 {
3668 let _ = sink.send(end).await;
3669 }
3670 }
3671 });
3672 streams.insert(key, task.abort_handle());
3673 Ok(Vec::new())
3674 }
3675
3676 async fn handle_activity_record(
3677 &self,
3678 ctx: &RouteCtx,
3679 frame: Frame,
3680 ) -> Result<Vec<Frame>, RouterError> {
3681 use subc_control::{
3682 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3683 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3684 };
3685 let result = async {
3686 let module = self
3687 .forwarding
3688 .module_id_for_connection(ctx.connection_id)
3689 .ok()
3690 .flatten()
3691 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3692 let request: ActivityRecordRequest =
3693 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3694 crate::activity::validate(&module, &request)?;
3695 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3696 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3697 let at_ms = std::time::SystemTime::now()
3698 .duration_since(std::time::UNIX_EPOCH)
3699 .unwrap_or_default()
3700 .as_millis() as u64;
3701 ledger.record(module, request, actor, at_ms).await
3702 }
3703 .await;
3704 match result {
3705 Ok(entry) => Ok(vec![control_response_body_frame(
3706 &frame,
3707 &ActivityRecordResponse { seq: entry.seq },
3708 "activity.record",
3709 )?]),
3710 Err(code) => Ok(vec![control_error_frame(
3711 &frame,
3712 code,
3713 "activity recording refused",
3714 )?]),
3715 }
3716 }
3717
3718 fn activity_actor(
3719 &self,
3720 connection: ConnectionId,
3721 module: &str,
3722 on: Option<&subc_control::ActivityOn>,
3723 ) -> Result<subc_control::ActivityActor, &'static str> {
3724 use crate::forwarding::{DataRoute, DataRouteState};
3725 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3726 let principal = |id: &str| Principal::Reserved {
3727 module_id: id.to_owned(),
3728 };
3729 let mut actor = ActivityActor {
3730 principal: principal(module),
3731 scope_owner: None,
3732 agent_id: None,
3733 run_id: None,
3734 flow_id: None,
3735 project_root: None,
3736 };
3737 let scope = match on {
3738 None => return Ok(actor),
3739 Some(ActivityOn::Scope(scope)) => {
3740 let selector = subc_protocol::scope::ScopeSelector::new(
3741 principal(module),
3742 scope.scope_ref.clone(),
3743 Some(scope.epoch),
3744 );
3745 self.scopes
3746 .read()
3747 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3748 .attribute(&principal(module), module, &selector, true)
3749 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3750 }
3751 Some(ActivityOn::Route(route)) => {
3752 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3753 .forwarding
3754 .lookup_data_route(connection, route.channel, route.epoch)
3755 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3756 else {
3757 return Err(ACTIVITY_ACTOR_UNKNOWN);
3758 };
3759 if binding.module_id != module {
3760 return Err(ACTIVITY_ACTOR_UNKNOWN);
3761 }
3762 actor.principal = binding.principal.clone();
3763 actor.project_root = binding
3764 .project_root
3765 .as_ref()
3766 .map(|root| root.as_path().to_path_buf());
3767 let Some(bound) = &binding.scope else {
3768 return Ok(actor);
3769 };
3770 let owner = principal(&bound.owner);
3771 let selector = subc_protocol::scope::ScopeSelector::new(
3772 owner.clone(),
3773 bound.scope_ref.clone(),
3774 Some(bound.tag.scope_epoch),
3775 );
3776 let admitted = self
3777 .scopes
3778 .read()
3779 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3780 .attribute(&owner, module, &selector, true)
3781 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3782 if admitted.tag != bound.tag {
3783 return Err(ACTIVITY_ACTOR_UNKNOWN);
3784 }
3785 admitted
3786 }
3787 };
3788 actor.scope_owner = Some(scope.stamp.owner);
3789 actor.agent_id = scope.stamp.attributes.agent_id;
3790 actor.run_id = scope.stamp.attributes.run_id;
3791 actor.flow_id = scope.stamp.attributes.flow_id;
3792 Ok(actor)
3793 }
3794
3795 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3796 self.refresh_capability_requirements();
3797 let mut counters = self.counters.snapshot();
3803 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3804 obj.insert(
3805 "activity_unparseable_lines".into(),
3806 ledger.read_errors().into(),
3807 );
3808 obj.insert(
3809 "activity_lifecycle_append_failures".into(),
3810 ledger.lifecycle_failures().into(),
3811 );
3812 obj.insert(
3813 "activity_started_digest_skips".into(),
3814 ledger.lifecycle_digest_skips().into(),
3815 );
3816 }
3817 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3818 self.forwarding.client_route_concentration(),
3819 counters.as_object_mut(),
3820 ) {
3821 obj.insert(
3822 "client_connections_with_routes".into(),
3823 connections_with_routes.into(),
3824 );
3825 obj.insert("max_routes_on_one_connection".into(), max.into());
3826 }
3827 if let (Some(open_breakers), Some(obj)) = (
3833 self.route_bind_breakers.open_snapshot(),
3834 counters.as_object_mut(),
3835 ) {
3836 obj.insert("route_bind_breakers_open".into(), open_breakers);
3837 }
3838 let response = ClientControlResponse::ServerDescribe {
3839 protocol_ver: PROTOCOL_VERSION,
3840 subc_ops: subc_ops(),
3841 capabilities: self.subc_capabilities.as_ref().to_vec(),
3842 connected_clients: self.connected_clients.count(),
3843 counters: Some(counters),
3844 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3845 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3846 capability_requirements: self.capability_requirement_statuses(),
3847 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3848 };
3849 Ok(vec![control_response_body_frame(
3850 &frame,
3851 &response,
3852 "ClientControlResponse::ServerDescribe",
3853 )?])
3854 }
3855
3856 fn handle_catalog_list(
3857 &self,
3858 frame: Frame,
3859 module_id: Option<String>,
3860 ) -> Result<Vec<Frame>, RouterError> {
3861 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3862 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3863 })?;
3864 let entries = modules
3865 .into_iter()
3866 .filter(|registration| {
3867 module_id
3868 .as_deref()
3869 .map(|wanted| registration.manifest.module_id == wanted)
3870 .unwrap_or(true)
3871 })
3872 .map(|registration| {
3873 let not_ready = self.not_ready_reason(®istration);
3874 let roles = registration.manifest.provides;
3875 CatalogEntry::new(
3876 registration.manifest.module_id,
3877 roles,
3878 registration.control_ops,
3879 )
3880 .with_ready(not_ready.is_none())
3881 .with_not_ready(not_ready)
3882 .with_module_version(Some(registration.manifest.module_version))
3883 .with_capabilities(registration.manifest.capabilities)
3884 .with_self_signals(registration.manifest.self_signals)
3885 .with_events(registration.manifest.events)
3886 .with_operation_delivery(registration.manifest.operation_delivery)
3887 })
3888 .collect();
3889 let response = ClientControlResponse::CatalogList {
3890 generation,
3891 modules: entries,
3892 subc_ops: subc_ops(),
3893 };
3894 Ok(vec![control_response_body_frame(
3895 &frame,
3896 &response,
3897 "ClientControlResponse::CatalogList",
3898 )?])
3899 }
3900
3901 fn route_open_principal(
3902 &self,
3903 frame: &Frame,
3904 consumer_identity: Option<ConsumerIdentity>,
3905 ) -> Result<Result<Principal, Frame>, RouterError> {
3906 let Some(consumer_identity) = consumer_identity else {
3907 return Ok(Ok(Principal::Direct));
3908 };
3909
3910 if self.supervisor.spawned_consumer_authorized(
3911 &consumer_identity.module_id,
3912 &consumer_identity.launch_nonce,
3913 ) {
3914 return Ok(Ok(Principal::Reserved {
3915 module_id: consumer_identity.module_id,
3916 }));
3917 }
3918
3919 Ok(Err(control_error_frame(
3920 frame,
3921 "bad_consumer_identity",
3922 format!(
3923 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3924 consumer_identity.module_id
3925 ),
3926 )?))
3927 }
3928
3929 fn route_open_refusal_frame(
3936 &self,
3937 ctx: &RouteCtx,
3938 frame: &Frame,
3939 module_id: &str,
3940 reason: &'static str,
3941 code: &'static str,
3942 message: impl Into<String>,
3943 ) -> Result<Frame, RouterError> {
3944 self.observe_route_open_refusal(ctx, module_id, reason, code);
3945 control_error_frame(frame, code, message.into())
3946 }
3947
3948 fn route_open_breaker_refusal_frame(
3966 &self,
3967 ctx: &RouteCtx,
3968 frame: &Frame,
3969 module_id: &str,
3970 consecutive_timeouts: u32,
3971 retry_in: Duration,
3972 probe_in_flight: bool,
3973 ) -> Result<Frame, RouterError> {
3974 self.counters
3975 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3976 debug!(
3977 target: "control",
3978 code = "module_timeout",
3979 module_id = ?module_id,
3980 connection_id = ctx.connection_id.get(),
3981 consecutive_timeouts,
3982 retry_in_ms = retry_in.as_millis() as u64,
3983 probe_in_flight,
3984 "route.open refused by open bind-relay breaker"
3985 );
3986 let detail = if probe_in_flight {
3993 "one test route is already being tried; retry once it settles".to_string()
3994 } else {
3995 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3996 };
3997 control_error_frame(
3998 frame,
3999 "module_timeout",
4000 format!(
4001 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4002 timed out in a row); {detail}; its established routes are unaffected"
4003 ),
4004 )
4005 }
4006
4007 fn observe_route_open_refusal(
4020 &self,
4021 ctx: &RouteCtx,
4022 module_id: &str,
4023 reason: &'static str,
4024 code: &'static str,
4025 ) {
4026 self.counters.increment_route_open_refused(code);
4027 info!(
4028 target: "control",
4029 code,
4030 reason,
4031 module_id = ?module_id,
4032 connection_id = ctx.connection_id.get(),
4033 "route.open refused"
4034 );
4035 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4036 self.route_outages.record_not_serving(module_id, reason);
4037 }
4038 }
4039
4040 fn observe_route_open_accept(
4085 &self,
4086 ctx: &RouteCtx,
4087 module_id: &str,
4088 principal: &str,
4089 scope: &str,
4090 ) {
4091 self.route_outages.record_accepted(module_id);
4092 self.counters.increment_route_open_accepted(principal);
4093 info!(
4094 target: "control",
4095 principal,
4096 module_id,
4097 connection_id = ctx.connection_id.get(),
4098 scope = %scope,
4099 "route.open accepted"
4100 );
4101 }
4102
4103 fn supervised_absent_route_open_refusal_frame(
4104 &self,
4105 ctx: &RouteCtx,
4106 frame: &Frame,
4107 module_id: &str,
4108 code: &'static str,
4109 status: &crate::supervise::ModuleStatus,
4110 ) -> Result<Frame, RouterError> {
4111 self.counters.increment_route_open_refused(code);
4112 info!(
4113 target: "control",
4114 code,
4115 reason = "supervised_not_registered",
4116 module_id = ?module_id,
4117 connection_id = ctx.connection_id.get(),
4118 state = %status.state,
4119 enabled = status.enabled,
4120 live = status.live,
4121 "route.open refused"
4122 );
4123 self.route_outages
4127 .record_not_serving(module_id, "supervised_not_registered");
4128 let message = if let Some(pid) = status.waiting_for_previous_pid {
4129 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4130 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4131 })
4132 } else {
4133 format!(
4134 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4135 status.state, status.enabled, status.live
4136 )
4137 };
4138 control_error_frame(frame, code, message)
4139 }
4140
4141 async fn handle_route_open(
4142 &self,
4143 ctx: &RouteCtx,
4144 frame: Frame,
4145 request: RouteOpenRequest,
4146 timings: &mut RouteOpenTimings,
4147 ) -> Result<Vec<Frame>, RouterError> {
4148 let RouteOpenRequest {
4149 target,
4150 mut identity,
4151 consumer_identity,
4152 consumer_capabilities,
4153 role_versions,
4154 admission_facts,
4155 scope,
4156 } = request;
4157 let target_module_id = target_module_id(&target).to_string();
4158 timings.module_id.clone_from(&target_module_id);
4159 if self
4160 .registry
4161 .get_module_by_connection(ctx.connection_id)
4162 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4163 .is_some()
4164 {
4165 return Ok(vec![control_error_frame(
4166 &frame,
4167 "invalid_request",
4168 "module connections cannot open client routes",
4169 )?]);
4170 }
4171 debug!(
4172 connection_id = ctx.connection_id.get(),
4173 corr = frame.header.corr,
4174 module_id = %target_module_id,
4175 "handling route.open"
4176 );
4177
4178 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4184 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4185 self.observe_route_open_refusal(
4186 ctx,
4187 &target_module_id,
4188 "invalid_role_versions",
4189 error_codes::INVALID_REQUEST,
4190 );
4191 return Ok(vec![control_error_body_frame(
4192 &frame,
4193 ErrorBody {
4194 code: error_codes::INVALID_REQUEST.to_string(),
4195 message: error.to_string(),
4196 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4197 },
4198 )?]);
4199 }
4200
4201 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4202 if status.waiting_for_previous_pid.is_some() {
4203 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4204 ctx,
4205 &frame,
4206 &target_module_id,
4207 "module_warming",
4208 &status,
4209 )?]);
4210 }
4211 }
4212
4213 let Some(registration) = self
4232 .registry
4233 .get_module(&target_module_id)
4234 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4235 else {
4236 if let Some((status, warming)) =
4237 self.supervisor_status(&target_module_id, frame.header.corr)?
4238 {
4239 if status.protocol == ModuleProtocol::None {
4246 return Ok(vec![self.route_open_refusal_frame(
4247 ctx,
4248 &frame,
4249 &target_module_id,
4250 "protocol_none",
4251 error_codes::MODULE_NO_PROTOCOL,
4252 format!(
4253 "module_id '{target_module_id}' is declared protocol: none; \
4254 it speaks no subc wire and serves no routes"
4255 ),
4256 )?]);
4257 }
4258 let code = if warming {
4259 "module_warming"
4260 } else {
4261 "target_unavailable"
4262 };
4263 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4264 ctx,
4265 &frame,
4266 &target_module_id,
4267 code,
4268 &status,
4269 )?]);
4270 }
4271 if let Some(removed_ago_ms) =
4272 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4273 {
4274 return Ok(vec![self.route_open_refusal_frame(
4275 ctx,
4276 &frame,
4277 &target_module_id,
4278 "removed",
4279 error_codes::MODULE_REMOVED,
4280 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4281 )?]);
4282 }
4283 return Ok(vec![self.route_open_refusal_frame(
4284 ctx,
4285 &frame,
4286 &target_module_id,
4287 "not_registered",
4288 error_codes::UNKNOWN_MODULE,
4289 format!("module_id '{target_module_id}' is not registered"),
4290 )?]);
4291 };
4292
4293 if !registration.ready {
4297 self.counters
4298 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4299 info!(
4300 target: "control",
4301 code = error_codes::MODULE_WARMING,
4302 module_id = ?target_module_id,
4303 connection_id = ctx.connection_id.get(),
4304 reason = "declared_not_ready",
4305 "route.open refused"
4306 );
4307 self.route_outages
4310 .record_not_serving(&target_module_id, "declared_not_ready");
4311 return Ok(vec![control_error_body_frame(
4312 &frame,
4313 ErrorBody {
4314 code: error_codes::MODULE_WARMING.to_string(),
4315 message: format!(
4316 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4317 ),
4318 detail: Some(serde_json::json!({
4319 "reason": "declared_not_ready"
4320 })),
4321 },
4322 )?]);
4323 }
4324
4325 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4344 self.counters
4345 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4346 info!(
4347 target: "control",
4348 code = error_codes::MODULE_WARMING,
4349 module_id = ?target_module_id,
4350 connection_id = ctx.connection_id.get(),
4351 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4352 capability = %capability,
4353 "route.open refused"
4354 );
4355 return Ok(vec![control_error_body_frame(
4356 &frame,
4357 ErrorBody {
4358 code: error_codes::MODULE_WARMING.to_string(),
4359 message: format!(
4360 "module_id '{target_module_id}' requires capability '{capability}', \
4361 which no registered module provides; retry"
4362 ),
4363 detail: Some(serde_json::json!({
4364 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4365 "capability": capability,
4366 })),
4367 },
4368 )?]);
4369 }
4370
4371 if !target_has_required_role(&target, ®istration.manifest.provides) {
4372 return Ok(vec![self.route_open_refusal_frame(
4373 ctx,
4374 &frame,
4375 &target_module_id,
4376 "role_not_provided",
4377 "target_unavailable",
4378 format!("module_id '{target_module_id}' does not provide the requested target"),
4379 )?]);
4380 }
4381
4382 if registration.state != ChannelState::Active {
4383 return Ok(vec![self.route_open_refusal_frame(
4384 ctx,
4385 &frame,
4386 &target_module_id,
4387 "registration_not_active",
4388 "target_unavailable",
4389 format!("module_id '{target_module_id}' is not active"),
4390 )?]);
4391 }
4392
4393 if self
4394 .forwarding
4395 .module_is_draining(&target_module_id)
4396 .map_err(RouterError::Forwarding)?
4397 {
4398 return Ok(vec![self.route_open_refusal_frame(
4399 ctx,
4400 &frame,
4401 &target_module_id,
4402 "reloading",
4403 "module_reloading",
4404 format!("module_id '{target_module_id}' is reloading"),
4405 )?]);
4406 }
4407
4408 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4409 process_liveness.process_live(&target_module_id) == Some(false)
4410 }) {
4411 if process_liveness.process_replacing(&target_module_id) {
4418 return Ok(vec![self.route_open_refusal_frame(
4419 ctx,
4420 &frame,
4421 &target_module_id,
4422 "reloading",
4423 "module_reloading",
4424 format!("module_id '{target_module_id}' is reloading"),
4425 )?]);
4426 }
4427 return Ok(vec![self.route_open_refusal_frame(
4428 ctx,
4429 &frame,
4430 &target_module_id,
4431 "supervisor_not_live",
4432 "target_unavailable",
4433 format!("module_id '{target_module_id}' is not live"),
4434 )?]);
4435 }
4436
4437 if !self
4438 .forwarding
4439 .has_live_module_connection(&target_module_id)
4440 .map_err(RouterError::Forwarding)?
4441 {
4442 return Ok(vec![self.route_open_refusal_frame(
4443 ctx,
4444 &frame,
4445 &target_module_id,
4446 "no_forwarding_connection",
4447 "target_unavailable",
4448 format!("module_id '{target_module_id}' has no live forwarding connection"),
4449 )?]);
4450 }
4451
4452 if let Some(error) =
4453 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4454 {
4455 self.observe_route_open_refusal(
4456 ctx,
4457 &target_module_id,
4458 "op_not_allowed",
4459 "op_not_allowed",
4460 );
4461 return Ok(vec![error]);
4462 }
4463
4464 let principal = match self.route_open_principal(&frame, consumer_identity)? {
4465 Ok(principal) => principal,
4466 Err(error) => {
4467 self.observe_route_open_refusal(
4468 ctx,
4469 &target_module_id,
4470 "bad_consumer_identity",
4471 "bad_consumer_identity",
4472 );
4473 return Ok(vec![error]);
4474 }
4475 };
4476 timings.principal = principal_label(&principal);
4477
4478 if let Principal::Reserved {
4482 module_id: opening_module_id,
4483 } = &principal
4484 {
4485 if let Some(opening_registration) = self
4486 .registry
4487 .get_module(opening_module_id)
4488 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4489 {
4490 if let Some(capability) =
4491 denied_capability(&opening_registration.manifest, ®istration.manifest)
4492 {
4493 warn!(
4494 opening_module_id,
4495 target_module_id,
4496 capability,
4497 "refusing route.open because an attested capability deny edge matches"
4498 );
4499 return Ok(vec![self.route_open_refusal_frame(
4500 ctx,
4501 &frame,
4502 &target_module_id,
4503 "capability_deny_edge",
4504 "capability_forbidden",
4505 format!(
4506 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4507 ),
4508 )?]);
4509 }
4510 }
4511 }
4512
4513 if admission_facts.is_some() {
4514 let carrier_matches = matches!(
4515 &principal,
4516 Principal::Reserved { module_id }
4517 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4518 );
4519 if !carrier_matches {
4520 return Ok(vec![self.route_open_refusal_frame(
4521 ctx,
4522 &frame,
4523 &target_module_id,
4524 "admission_facts_carrier_not_permitted",
4525 "admission_facts_not_permitted",
4526 "admission facts may only be carried by the configured reserved module",
4527 )?]);
4528 }
4529
4530 let target_allowed = self
4531 .admission_facts_targets
4532 .as_ref()
4533 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4534 if !target_allowed {
4535 return Ok(vec![self.route_open_refusal_frame(
4536 ctx,
4537 &frame,
4538 &target_module_id,
4539 "admission_facts_target_not_listed",
4540 "admission_facts_target_not_allowed",
4541 format!(
4542 "admission facts are not permitted for target module_id '{target_module_id}'"
4543 ),
4544 )?]);
4545 }
4546
4547 }
4551
4552 let (bound_scope, mut scope_stamp) = match &scope {
4558 None => (None, None),
4559 Some(selector) => {
4560 let owner_configured = match &selector.owner {
4561 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4567 _ => false,
4568 };
4569 let mut scopes = self
4570 .scopes
4571 .write()
4572 .unwrap_or_else(|poisoned| poisoned.into_inner());
4573 let admitted = scopes
4574 .admit(&principal, &target_module_id, selector, owner_configured)
4575 .and_then(|admission| {
4576 crate::scopes::check_target_flow_support(
4577 &admission.stamp,
4578 &target_module_id,
4579 registration.manifest.capabilities.as_ref(),
4580 )?;
4581 crate::scopes::check_target_agent_run_support(
4582 &admission.stamp,
4583 &target_module_id,
4584 registration.manifest.capabilities.as_ref(),
4585 )?;
4586 let bound = scopes.reserve_bind(
4587 &admission,
4588 &principal,
4589 ctx.connection_id,
4590 selector.hold_presence,
4591 );
4592 Ok((admission, bound))
4593 });
4594 match admitted {
4595 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4596 Err(refusal) => {
4597 return Ok(vec![self.route_open_refusal_frame(
4598 ctx,
4599 &frame,
4600 &target_module_id,
4601 refusal.code,
4602 refusal.code,
4603 refusal.message,
4604 )?]);
4605 }
4606 }
4607 }
4608 };
4609
4610 timings.enter(RouteOpenPhase::ProjectRoot);
4632 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4633 Ok(project_root) => project_root,
4634 Err(err) => {
4635 return Ok(vec![control_error_frame(
4636 &frame,
4637 "invalid_project_root",
4638 err.to_string(),
4639 )?])
4640 }
4641 };
4642 timings.enter(RouteOpenPhase::Admission);
4643 identity.project_root = project_root.as_path().to_path_buf();
4644
4645 let _concurrency_guard = match self
4656 .route_bind_concurrency
4657 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4658 {
4659 Ok(guard) => guard,
4660 Err(in_flight) => {
4661 return Ok(vec![self.route_open_target_capacity_refusal(
4662 ctx,
4663 &frame,
4664 &target_module_id,
4665 in_flight,
4666 )?]);
4667 }
4668 };
4669
4670 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4673 RouteBindAdmission::Admitted { guard, probe } => {
4674 if probe {
4675 info!(
4676 module_id = %target_module_id,
4677 connection_id = ctx.connection_id.get(),
4678 "route.bind breaker half-open: admitting one probe"
4679 );
4680 }
4681 guard
4682 }
4683 RouteBindAdmission::Refused {
4684 consecutive_timeouts,
4685 retry_in,
4686 probe_in_flight,
4687 } => {
4688 return Ok(vec![self.route_open_breaker_refusal_frame(
4689 ctx,
4690 &frame,
4691 &target_module_id,
4692 consecutive_timeouts,
4693 retry_in,
4694 probe_in_flight,
4695 )?]);
4696 }
4697 };
4698
4699 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4704 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4705 timings.enter(RouteOpenPhase::EgressReservation);
4706 let pending = match self
4707 .forwarding
4708 .begin_route_bind_relay_for(
4709 ctx.connection_id,
4710 ctx.egress.clone(),
4711 response_version(&frame),
4712 frame.header.corr,
4713 &target_module_id,
4714 principal.clone(),
4715 bound_scope,
4716 Some(project_root),
4717 relay_deadline,
4718 )
4719 .await
4720 {
4721 Ok(pending) => pending,
4722 Err(err) => {
4723 return Ok(vec![self.route_open_refusal_frame(
4724 ctx,
4725 &frame,
4726 &target_module_id,
4727 "relay_reservation_failed",
4728 forwarding_error_code(&err),
4729 err.to_string(),
4730 )?])
4731 }
4732 };
4733 timings.enter(RouteOpenPhase::Admission);
4734 let crate::forwarding::PendingRouteBindRelay {
4735 endpoint,
4736 module_sink,
4737 negotiated_ver,
4738 client_channel,
4739 client_epoch,
4740 module_channel,
4741 module_epoch,
4742 corr: relay_corr,
4743 receiver,
4744 } = pending;
4745 let mut reservation =
4746 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4747
4748 let relay_registration = self
4753 .registry
4754 .get_module_by_connection(endpoint.connection_id)
4755 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4756 let relay_capabilities = relay_registration
4757 .as_ref()
4758 .and_then(|registration| registration.manifest.capabilities.as_ref());
4759 if let Some(selector) = &scope {
4760 if let Err(refusal) = self
4763 .scopes
4764 .read()
4765 .unwrap_or_else(|poisoned| poisoned.into_inner())
4766 .check_bind_proof(&principal, selector)
4767 {
4768 reservation.release_and_disarm();
4769 return Ok(vec![self.route_open_refusal_frame(
4770 ctx,
4771 &frame,
4772 &target_module_id,
4773 refusal.code,
4774 refusal.code,
4775 refusal.message,
4776 )?]);
4777 }
4778 }
4779 if let Some(stamp) = scope_stamp
4780 .as_ref()
4781 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4782 {
4783 if let Err(refusal) = crate::scopes::check_target_flow_support(
4784 stamp,
4785 &target_module_id,
4786 relay_capabilities,
4787 )
4788 .and_then(|()| {
4789 crate::scopes::check_target_agent_run_support(
4790 stamp,
4791 &target_module_id,
4792 relay_capabilities,
4793 )
4794 }) {
4795 reservation.release_and_disarm();
4796 return Ok(vec![self.route_open_refusal_frame(
4797 ctx,
4798 &frame,
4799 &target_module_id,
4800 refusal.code,
4801 refusal.code,
4802 refusal.message,
4803 )?]);
4804 }
4805 }
4806
4807 if !relay_capabilities.is_some_and(|caps| {
4810 caps.provides
4811 .iter()
4812 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
4813 }) {
4814 if let Some(stamp) = scope_stamp.as_mut() {
4815 stamp.attributes.on_behalf_of_agent = None;
4816 }
4817 }
4818 let call_mode = if relay_capabilities
4819 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4820 {
4821 Some(derive_call_mode(scope_stamp.as_ref()))
4822 } else {
4823 if let Some(stamp) = scope_stamp.as_mut() {
4824 stamp.expires_at_ms = None;
4825 }
4826 None
4827 };
4828
4829 debug!(
4830 connection_id = ctx.connection_id.get(),
4831 client_channel,
4832 client_epoch,
4833 module_channel,
4834 module_epoch,
4835 "reserved route handle pair"
4836 );
4837 let route_principal_label = principal_label(&principal);
4840 let scope_label = scope_stamp
4841 .as_ref()
4842 .map_or_else(|| "none".to_string(), route_open_scope_label);
4843 let relay = ModuleControlRequest::RouteBind {
4844 route_channel: module_channel,
4845 epoch: module_epoch,
4846 target,
4847 identity,
4848 principal: Some(principal),
4849 consumer_capabilities,
4850 role_versions,
4851 admission_facts,
4852 scope: scope_stamp,
4853 call_mode,
4854 };
4855 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4856 RouterError::backend(
4857 0,
4858 frame.header.corr,
4859 format!("failed to encode route.bind request: {err}"),
4860 )
4861 })?;
4862 let relay_frame = Frame::build_with_version(
4863 negotiated_ver,
4864 FrameType::Request,
4865 control_flags(),
4866 0,
4867 0,
4868 relay_corr,
4869 relay_body,
4870 )
4871 .map_err(RouterError::FrameBuild)?;
4872
4873 timings.enter(RouteOpenPhase::BindSend);
4874 let sent = module_sink.send(relay_frame).await;
4875 timings.enter(RouteOpenPhase::Admission);
4876 if let Err(err) = sent {
4877 reservation.release_and_disarm();
4878 return Ok(vec![self.route_open_refusal_frame(
4879 ctx,
4880 &frame,
4881 &target_module_id,
4882 "relay_send_failed",
4883 "target_unavailable",
4884 err.to_string(),
4885 )?]);
4886 }
4887
4888 #[cfg(test)]
4889 if let Some(hold) = &self.route_bind_relay_sent_hold {
4890 hold.wait().await;
4894 hold.wait().await;
4895 }
4896
4897 if self
4902 .forwarding
4903 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4904 .map_err(RouterError::Forwarding)?
4905 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4906 {
4907 self.send_abandoned_route_bind_goodbye(
4908 &module_sink,
4909 negotiated_ver,
4910 module_channel,
4911 module_epoch,
4912 );
4913 }
4914
4915 timings.enter(RouteOpenPhase::BindAckWait);
4916 let outcome = timeout_at(relay_deadline, receiver).await;
4917 timings.enter(RouteOpenPhase::Admission);
4918 match outcome {
4919 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4920 reservation.disarm();
4921 if breaker.record_accepted() {
4922 info!(
4923 module_id = %target_module_id,
4924 "route.bind breaker closed: the probe was accepted"
4925 );
4926 }
4927 self.observe_route_open_accept(
4928 ctx,
4929 &target_module_id,
4930 &route_principal_label,
4931 &scope_label,
4932 );
4933 Ok(Vec::new())
4934 }
4935 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4936 reservation.release_and_disarm();
4937 breaker.record_inconclusive();
4941 let scope_code = match body.code.as_str() {
4945 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4946 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4947 _ => None,
4948 };
4949 if let Some(code) = scope_code {
4950 self.observe_route_open_refusal(
4951 ctx,
4952 &target_module_id,
4953 "scope_changed_before_commit",
4954 code,
4955 );
4956 return Ok(vec![control_error_body_frame(&frame, body)?]);
4957 }
4958 self.counters
4959 .increment_route_open_refused("module_rejected");
4960 info!(
4961 target: "control",
4962 code = "module_rejected",
4963 module_code = ?body.code,
4964 module_id = ?target_module_id,
4965 connection_id = ctx.connection_id.get(),
4966 "route.open refused"
4967 );
4968 Ok(vec![control_error_body_frame(&frame, body)?])
4969 }
4970 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4971 reservation.release_and_disarm();
4972 breaker.record_inconclusive();
4973 tracing::warn!(
4980 module_id = %target_module_id,
4981 "route.bind relay abandoned: {message}"
4982 );
4983 Ok(vec![self.route_open_refusal_frame(
4984 ctx,
4985 &frame,
4986 &target_module_id,
4987 "relay_abandoned",
4988 "target_unavailable",
4989 message,
4990 )?])
4991 }
4992 Ok(Err(_)) => {
4993 reservation.release_and_disarm();
4994 breaker.record_inconclusive();
4995 Ok(vec![self.route_open_refusal_frame(
4996 ctx,
4997 &frame,
4998 &target_module_id,
4999 "relay_waiter_canceled",
5000 "target_unavailable",
5001 "route.bind relay waiter was canceled before the module responded",
5002 )?])
5003 }
5004 Err(_) => {
5005 reservation.release_and_disarm();
5006 if let Some(opened) = breaker.record_timeout(
5010 self.route_bind_breaker_threshold,
5011 self.route_bind_breaker_cooldown,
5012 ) {
5013 warn!(
5014 module_id = %target_module_id,
5015 consecutive_timeouts = opened.consecutive_timeouts,
5016 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5017 reopened_after_probe = opened.reopened_after_probe,
5018 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5019 );
5020 }
5021 tracing::warn!(
5029 module_id = %target_module_id,
5030 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5031 "route.bind relay timed out: module did not ack within budget"
5032 );
5033 Ok(vec![self.route_open_refusal_frame(
5034 ctx,
5035 &frame,
5036 &target_module_id,
5037 "relay_timed_out",
5038 "module_timeout",
5039 format!(
5040 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5041 route_bind_relay_timeout
5042 ),
5043 )?])
5044 }
5045 }
5046 }
5047
5048 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5049 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5050 snapshot: self.supervisor.spawn_snapshot(),
5051 };
5052 Ok(vec![control_response_body_frame(
5053 &frame,
5054 &response,
5055 "ClientControlResponse::SupervisorSpawnSnapshot",
5056 )?])
5057 }
5058
5059 fn handle_supervisor_spawn_subscribe(
5060 &self,
5061 ctx: &RouteCtx,
5062 frame: Frame,
5063 since: Option<SpawnCursor>,
5064 ) -> Result<Vec<Frame>, RouterError> {
5065 match self.supervisor.subscribe_spawns(
5066 ctx.connection_id,
5067 frame.header.corr,
5068 response_version(&frame),
5069 since,
5070 ctx.egress.clone(),
5071 ) {
5072 Ok(()) => Ok(Vec::new()),
5073 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5074 Ok(vec![control_error_body_frame(
5075 &frame,
5076 ErrorBody {
5077 code: "spawn_cursor_incarnation_mismatch".to_string(),
5078 message: "spawn cursor belongs to a different daemon incarnation"
5079 .to_string(),
5080 detail: Some(serde_json::json!({
5081 "current_daemon_incarnation": current
5082 })),
5083 },
5084 )?])
5085 }
5086 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5087 &frame,
5088 ErrorBody {
5089 code: "spawn_cursor_too_old".to_string(),
5090 message: "spawn cursor predates the retained event ring".to_string(),
5091 detail: Some(serde_json::json!({
5092 "oldest_retained_cursor": oldest
5093 })),
5094 },
5095 )?]),
5096 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5097 0,
5098 frame.header.corr,
5099 format!("failed to open supervisor spawn subscription: {error}"),
5100 )),
5101 }
5102 }
5103
5104 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5105 let generation = self
5106 .registry
5107 .generation()
5108 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5109 let mut modules = Vec::new();
5110 for module in self.supervisor.list() {
5111 let status = module.status_for_control("list").map_err(|err| {
5112 RouterError::backend(
5113 0,
5114 frame.header.corr,
5115 format!("failed to read supervisor status: {err}"),
5116 )
5117 })?;
5118 let (configured, _) = module.configuration().map_err(|err| {
5119 RouterError::backend(
5120 0,
5121 frame.header.corr,
5122 format!("failed to read module configuration: {err}"),
5123 )
5124 })?;
5125 let image = module.running_image_agreement().await;
5127 let resources = Some(module.child_resource_usage());
5130 let mut verdict =
5131 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5132 verdict.spec = Some(launch_spec_agreement(
5133 &configured,
5134 status.process_alive,
5135 status.launch_spec_digest.as_ref(),
5136 ));
5137 let pending_reload = Some(verdict);
5138 modules.push(
5141 SupervisorEntry::new(
5142 status.module_id,
5143 status.state.to_string(),
5144 status.enabled,
5145 status.live,
5146 status.health.status,
5147 )
5148 .with_launch_nonce_env(Some(!cfg!(unix)))
5149 .with_launch_nonce_source(status.launch_nonce_source)
5150 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5151 .with_protocol(status.protocol)
5152 .with_pending_reload(pending_reload)
5153 .with_last_probe_ms(status.health.last_probe_ms)
5154 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5155 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5156 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5157 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5158 .with_restart_count(Some(status.restart_count))
5159 .with_max_restarts(Some(status.max_restarts))
5160 .with_lifetime_restarts(Some(status.lifetime_restarts))
5161 .with_spawn_generation(Some(status.spawn_generation))
5162 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5163 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5164 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5165 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5166 .with_resources(resources),
5167 );
5168 }
5169 let response = ClientControlResponse::SupervisorList {
5170 generation,
5171 modules,
5172 };
5173 Ok(vec![control_response_body_frame(
5174 &frame,
5175 &response,
5176 "ClientControlResponse::SupervisorList",
5177 )?])
5178 }
5179
5180 fn handle_supervisor_stderr_tail(
5181 &self,
5182 frame: Frame,
5183 module_id: String,
5184 max_lines: Option<u32>,
5185 max_bytes: Option<u32>,
5186 ) -> Result<Vec<Frame>, RouterError> {
5187 let Some(module) = self.supervisor.get(&module_id) else {
5188 return Ok(vec![control_error_frame(
5189 &frame,
5190 "unknown_module",
5191 format!("module_id '{module_id}' is not supervised"),
5192 )?]);
5193 };
5194
5195 let snapshot = module.stderr_tail(
5196 max_lines.map(|value| value as usize),
5197 max_bytes.map(|value| value as usize),
5198 );
5199
5200 let response = ClientControlResponse::SupervisorStderrTail {
5201 module_id,
5202 tail: StderrTail {
5203 capture: match snapshot.capture {
5204 CaptureState::Captured => StderrCaptureState::Captured,
5205 CaptureState::Incomplete { reason } => {
5206 StderrCaptureState::Incomplete { reason }
5207 }
5208 CaptureState::NotCaptured { reason } => {
5209 StderrCaptureState::NotCaptured { reason }
5210 }
5211 },
5212 entries: snapshot
5213 .entries
5214 .into_iter()
5215 .map(|entry| match entry {
5216 TailEntry::Line {
5217 text,
5218 truncated,
5219 at_ms,
5220 } => StderrTailEntry::Line {
5221 text,
5222 truncated,
5223 at_ms,
5224 },
5225 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5226 })
5227 .collect(),
5228 dropped_lines: snapshot.dropped_lines,
5229 },
5230 };
5231 Ok(vec![control_response_body_frame(
5232 &frame,
5233 &response,
5234 "ClientControlResponse::SupervisorStderrTail",
5235 )?])
5236 }
5237
5238 async fn handle_supervisor_terminals(
5239 &self,
5240 frame: Frame,
5241 module_id: String,
5242 ) -> Result<Vec<Frame>, RouterError> {
5243 let Some(module) = self.supervisor.get(&module_id) else {
5244 return Ok(vec![control_error_frame(
5245 &frame,
5246 "unknown_module",
5247 format!("module_id '{module_id}' is not supervised"),
5248 )?]);
5249 };
5250
5251 let terminals = module
5254 .read_durable_terminal_history()
5255 .await
5256 .map_err(|error| {
5257 RouterError::backend(
5258 0,
5259 frame.header.corr,
5260 format!("failed to read terminal history: {error}"),
5261 )
5262 })?;
5263 let response = ClientControlResponse::SupervisorTerminals {
5264 module_id,
5265 terminals,
5266 };
5267 Ok(vec![control_response_body_frame(
5268 &frame,
5269 &response,
5270 "ClientControlResponse::SupervisorTerminals",
5271 )?])
5272 }
5273
5274 fn handle_supervisor_routes(
5275 &self,
5276 frame: Frame,
5277 module_id: Option<String>,
5278 ) -> Result<Vec<Frame>, RouterError> {
5279 let modules = self
5280 .forwarding
5281 .route_census(module_id.as_deref())
5282 .map_err(RouterError::Forwarding)?
5283 .into_iter()
5284 .map(|(module_id, routes)| SupervisorRouteModule {
5285 module_id,
5286 routes: routes
5287 .into_iter()
5288 .map(|route| SupervisorRoute {
5289 consumer: match route.principal {
5290 Principal::Reserved { module_id } => {
5291 SupervisorRouteConsumer::Reserved { module_id }
5292 }
5293 Principal::Direct | Principal::Unverified => {
5294 SupervisorRouteConsumer::Direct {
5295 connection_id: route.goodbye_target.connection_id.get(),
5296 }
5297 }
5298 },
5299 age_ms: Instant::now()
5300 .saturating_duration_since(route.bound_at)
5301 .as_millis()
5302 .try_into()
5303 .unwrap_or(u64::MAX),
5304 draining: route.draining,
5305 drain_reason: route.drain_reason,
5306 })
5307 .collect(),
5308 })
5309 .collect();
5310 let response = ClientControlResponse::SupervisorRoutes { modules };
5311 Ok(vec![control_response_body_frame(
5312 &frame,
5313 &response,
5314 "ClientControlResponse::SupervisorRoutes",
5315 )?])
5316 }
5317
5318 async fn handle_supervisor_provenance(
5319 &self,
5320 frame: Frame,
5321 module_id: Option<String>,
5322 ) -> Result<Vec<Frame>, RouterError> {
5323 let mut selected = if let Some(module_id) = module_id {
5324 let Some(module) = self.supervisor.get(&module_id) else {
5325 return Ok(vec![control_error_frame(
5326 &frame,
5327 "unknown_module",
5328 format!("module_id '{module_id}' is not supervised"),
5329 )?]);
5330 };
5331 vec![module]
5332 } else {
5333 self.supervisor.list()
5334 };
5335
5336 let mut modules = Vec::with_capacity(selected.len());
5337 for module in selected.drain(..) {
5338 let (status, observed_image) = module
5339 .status_and_running_image_agreement()
5340 .await
5341 .map_err(|err| {
5342 RouterError::backend(
5343 0,
5344 frame.header.corr,
5345 format!("failed to read supervisor status: {err}"),
5346 )
5347 })?;
5348 let module_declared = self
5349 .registry
5350 .get_module(&status.module_id)
5351 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5352 .and_then(|registration| registration.manifest.provenance)
5353 .map(|build| ModuleDeclaredProvenance::Reported { build })
5354 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5355 #[cfg(test)]
5356 let running_image = match &self.provenance_probe_override {
5357 Some(result) => result.clone(),
5358 None => observed_image,
5359 };
5360 #[cfg(not(test))]
5361 let running_image = observed_image;
5362 modules.push(SupervisorModuleProvenance {
5363 module_id: status.module_id,
5364 module_declared,
5365 daemon_observed: SupervisorObservedProcess {
5366 launch_nonce_source: status.launch_nonce_source,
5367 pid: status.pid,
5368 spawned_at_ms: status.spawned_at_ms,
5369 spawned_from: status.spawned_from,
5370 running_image,
5371 },
5372 });
5373 }
5374 let daemon = SupervisorDaemonProvenance {
5375 daemon_build: self.daemon_provenance.build.clone(),
5376 daemon_observed: DaemonObservedProcess {
5377 pid: self.daemon_provenance.pid,
5378 started_at_ms: self
5379 .daemon_provenance
5380 .start_clock
5381 .map(|clock| clock.started_at_ms())
5382 .or(self.daemon_provenance.started_at_ms),
5383 running_image: self
5384 .daemon_provenance
5385 .probe
5386 .observe(
5387 self.daemon_provenance.pid,
5388 self.daemon_provenance.executable_path.as_deref(),
5389 self.daemon_provenance.executable_identity.clone(),
5390 self.daemon_provenance.process_start_time,
5391 )
5392 .await,
5393 },
5394 };
5395 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5396 Ok(vec![control_response_body_frame(
5397 &frame,
5398 &response,
5399 "ClientControlResponse::SupervisorProvenance",
5400 )?])
5401 }
5402
5403 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5404 self.refresh_capability_requirements();
5405 let generation = self
5406 .registry
5407 .generation()
5408 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5409 let modules = self
5410 .supervisor
5411 .list()
5412 .into_iter()
5413 .map(|module| {
5414 let status = module.status_for_control("health").map_err(|err| {
5415 RouterError::backend(
5416 0,
5417 frame.header.corr,
5418 format!("failed to read supervisor health: {err}"),
5419 )
5420 })?;
5421 let module_id = status.module_id;
5422 let capability_detail = self
5423 .capability_evaluator
5424 .required_problem_detail(&module_id);
5425 Ok(SupervisorHealthEntry {
5426 module_id,
5427 status: status.health.status,
5428 detail: append_capability_problem_detail(
5429 status.health.detail,
5430 capability_detail,
5431 ),
5432 metrics: status.health.metrics,
5433 consecutive_failures: status.health.consecutive_failures,
5434 late_answer_count: status.health.late_answer_count,
5435 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5436 last_action: status.health.last_action,
5437 last_action_ms: status.health.last_action_ms,
5438 last_probe_ms: status.health.last_probe_ms,
5439 })
5440 })
5441 .collect::<Result<Vec<_>, RouterError>>()?;
5442 let response = ClientControlResponse::SupervisorHealth {
5443 generation,
5444 modules,
5445 };
5446 Ok(vec![control_response_body_frame(
5447 &frame,
5448 &response,
5449 "ClientControlResponse::SupervisorHealth",
5450 )?])
5451 }
5452
5453 async fn handle_supervisor_restart(
5454 &self,
5455 frame: Frame,
5456 module_id: String,
5457 drain_timeout_ms: Option<u64>,
5458 requester: Principal,
5459 ) -> Result<Vec<Frame>, RouterError> {
5460 let operation_lock = self.supervisor.operation_lock();
5461 let _operation_guard = operation_lock.lock().await;
5462 let Some(module) = self.supervisor.get(&module_id) else {
5463 return Ok(vec![control_error_frame(
5464 &frame,
5465 "unknown_module",
5466 format!("module_id '{module_id}' is not supervised"),
5467 )?]);
5468 };
5469
5470 self.route_outages.mark_operator_action(&module_id);
5471 if let Err(err) = module
5472 .restart_requested(drain_timeout_ms, Some(requester))
5473 .await
5474 {
5475 self.route_outages
5476 .operator_action_ended_unrefused(&module_id);
5477 let (code, message) = match err {
5478 crate::supervise::SuperviseError::Disabled { .. } => {
5479 ("module_disabled", err.to_string())
5480 }
5481 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5482 ("swap_in_progress", err.to_string())
5483 }
5484 _ => (
5485 "target_unavailable",
5486 format!("failed to restart module_id '{module_id}': {err}"),
5487 ),
5488 };
5489 return Ok(vec![control_error_frame(&frame, code, message)?]);
5490 }
5491
5492 let response = ClientControlResponse::SupervisorAck {
5493 module_id,
5494 applied: true,
5495 };
5496 Ok(vec![control_response_body_frame(
5497 &frame,
5498 &response,
5499 "ClientControlResponse::SupervisorAck",
5500 )?])
5501 }
5502
5503 async fn handle_supervisor_swap(
5507 &self,
5508 frame: Frame,
5509 module_id: String,
5510 ready_timeout_ms: Option<u64>,
5511 requester: Principal,
5512 ) -> Result<Vec<Frame>, RouterError> {
5513 let module = {
5520 let operation_lock = self.supervisor.operation_lock();
5521 let _operation_guard = operation_lock.lock().await;
5522 self.supervisor.get(&module_id)
5523 };
5524 let Some(module) = module else {
5525 return Ok(vec![control_error_frame(
5526 &frame,
5527 "unknown_module",
5528 format!("module_id '{module_id}' is not supervised"),
5529 )?]);
5530 };
5531
5532 self.route_outages.mark_operator_action(&module_id);
5533 if let Err(err) = module
5534 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5535 .await
5536 {
5537 self.route_outages
5538 .operator_action_ended_unrefused(&module_id);
5539 use crate::supervise::SuperviseError;
5540 let message = err.to_string();
5541 let error = match err {
5542 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5543 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5544 code: "swap_refused".to_string(),
5545 message,
5546 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5547 },
5548 SuperviseError::SwapFailed {
5549 arm,
5550 candidate_exit,
5551 ..
5552 } => ErrorBody {
5553 code: "swap_failed".to_string(),
5554 message,
5555 detail: Some(serde_json::json!({
5556 "arm": arm.as_str(),
5557 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5558 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5559 })),
5560 },
5561 _ => ErrorBody::new(
5562 "target_unavailable",
5563 format!("failed to swap module_id '{module_id}': {message}"),
5564 ),
5565 };
5566 return Ok(vec![control_error_body_frame(&frame, error)?]);
5567 }
5568 self.route_outages
5572 .operator_action_ended_unrefused(&module_id);
5573
5574 let response = ClientControlResponse::SupervisorAck {
5575 module_id,
5576 applied: true,
5577 };
5578 Ok(vec![control_response_body_frame(
5579 &frame,
5580 &response,
5581 "ClientControlResponse::SupervisorAck",
5582 )?])
5583 }
5584
5585 async fn handle_supervisor_reload(
5586 &self,
5587 frame: Frame,
5588 module_id: String,
5589 requester: Principal,
5590 ) -> Result<Vec<Frame>, RouterError> {
5591 let operation_lock = self.supervisor.operation_lock();
5592 let _operation_guard = operation_lock.lock().await;
5593 let Some(module) = self.supervisor.get(&module_id) else {
5594 return Ok(vec![control_error_frame(
5595 &frame,
5596 "unknown_module",
5597 format!("module_id '{module_id}' is not supervised"),
5598 )?]);
5599 };
5600
5601 self.route_outages.mark_operator_action(&module_id);
5602 if let Err(err) = module.reload_requested(Some(requester)).await {
5603 self.route_outages
5604 .operator_action_ended_unrefused(&module_id);
5605 let (code, message) = match err {
5606 crate::supervise::SuperviseError::Disabled { .. } => {
5607 ("module_disabled", err.to_string())
5608 }
5609 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5610 ("swap_in_progress", err.to_string())
5611 }
5612 _ => (
5613 "reload_failed",
5614 format!("failed to reload module_id '{module_id}': {err}"),
5615 ),
5616 };
5617 return Ok(vec![control_error_frame(&frame, code, message)?]);
5618 }
5619
5620 let response = ClientControlResponse::SupervisorAck {
5621 module_id,
5622 applied: true,
5623 };
5624 Ok(vec![control_response_body_frame(
5625 &frame,
5626 &response,
5627 "ClientControlResponse::SupervisorAck",
5628 )?])
5629 }
5630
5631 async fn handle_supervisor_rescan(
5632 &self,
5633 frame: Frame,
5634 preview: bool,
5635 requester: Principal,
5636 ) -> Result<Vec<Frame>, RouterError> {
5637 let Some(context) = self.rescan.clone() else {
5638 return Ok(vec![control_error_frame(
5639 &frame,
5640 "rescan_unavailable",
5641 "the daemon was not started with a reloadable config path".to_string(),
5642 )?]);
5643 };
5644
5645 let operation_lock = self.supervisor.operation_lock();
5646 let _operation_guard = operation_lock.lock().await;
5647 let loaded = match crate::daemon_config::load_with_activity(&context.config_path) {
5648 Ok(config) => config,
5649 Err(err) => {
5650 return Ok(vec![control_error_frame(
5651 &frame,
5652 "invalid_daemon_config",
5653 format!("supervisor rescan rejected daemon config: {err}"),
5654 )?])
5655 }
5656 };
5657 let Some((config, activity_config)) = loaded else {
5665 return Ok(vec![control_error_frame(
5666 &frame,
5667 "invalid_daemon_config",
5668 format!(
5669 "daemon config not found at {}; refusing to rescan (an absent config would \
5670 retire every supervised module)",
5671 context.config_path.display()
5672 ),
5673 )?]);
5674 };
5675 let (
5676 configured_port,
5677 storage_config,
5678 admission_facts_carrier_module_id,
5679 admission_facts_targets,
5680 scope_authority_owners,
5681 privileged_operator_prompt,
5682 modules,
5683 reserved_capabilities,
5684 ) = (
5685 config.port,
5686 config.storage,
5687 config.admission_facts_carrier_module_id,
5688 config.admission_facts_targets,
5689 config.scope_authority_owners,
5690 config.privileged_operator_prompt,
5691 config.modules,
5692 config.reserved_capabilities,
5693 );
5694
5695 let mut restart_required = Vec::new();
5704 for section in RestartRequiredSection::ALL {
5705 let changed = match section {
5706 RestartRequiredSection::Port => configured_port != context.configured_port,
5707 RestartRequiredSection::Storage => storage_config != context.storage_config,
5708 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5709 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5710 }
5711 RestartRequiredSection::AdmissionFactsTargets => {
5712 admission_facts_targets != context.admission_facts_targets
5713 }
5714 RestartRequiredSection::ScopeAuthorityOwners => {
5715 scope_authority_owners != context.scope_authority_owners
5716 }
5717 RestartRequiredSection::PrivilegedOperatorPrompt => {
5718 privileged_operator_prompt != context.privileged_operator_prompt
5719 }
5720 };
5721 if changed {
5722 restart_required.push(section.label().to_string());
5723 }
5724 }
5725 if !restart_required.is_empty() {
5726 warn!(
5727 config_path = %context.config_path.display(),
5728 sections = %restart_required.join(", "),
5729 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5730 );
5731 }
5732
5733 for configured in &modules {
5734 if let Err(err) = validate_spec(&configured.module_spec()) {
5735 return Ok(vec![control_error_frame(
5736 &frame,
5737 "invalid_daemon_config",
5738 format!("supervisor rescan rejected daemon config: {err}"),
5739 )?]);
5740 }
5741 }
5742
5743 let configured_capabilities = modules
5744 .iter()
5745 .map(|module| (module.module_id.clone(), module.enabled))
5746 .collect::<Vec<_>>();
5747 let preview_capability_warnings = if preview {
5748 let (_, registrations) = self.runtime_capability_snapshot()?;
5749 let current_modules = self
5750 .supervisor
5751 .list()
5752 .into_iter()
5753 .map(|module| module.module_id().to_string())
5754 .collect::<BTreeSet<_>>();
5755 let resulting_modules = configured_capabilities.clone();
5756 let removed = current_modules
5757 .into_iter()
5758 .filter(|module_id| {
5759 !resulting_modules
5760 .iter()
5761 .any(|(configured_id, _)| configured_id == module_id)
5762 })
5763 .collect::<Vec<_>>();
5764 self.capability_evaluator.preview_removal_warnings(
5765 resulting_modules,
5766 &removed,
5767 ®istrations,
5768 )
5769 } else {
5770 Vec::new()
5771 };
5772 let result = match self
5773 .reconcile_supervised_modules(&context.supervisor, modules, preview, requester.clone())
5774 .await
5775 {
5776 Ok(result) => result,
5777 Err(message) => {
5778 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5779 }
5780 };
5781 if !preview {
5782 if let Some(ledger) = &self.activity {
5783 ledger.module_rescanned(&result.added, &result.removed, requester);
5784 }
5785 if let Err(code) = self.configure_activity(activity_config).await {
5786 warn!(
5787 code,
5788 "activity retention could not be applied during rescan"
5789 );
5790 }
5791 self.capability_evaluator
5792 .configure(configured_capabilities, reserved_capabilities);
5793 self.capability_evaluator.wake_deadline_loop();
5794 self.refresh_capability_requirements();
5795 }
5796 let mut result = result;
5797 result.restart_required = restart_required;
5798 result.capability_warnings = preview_capability_warnings;
5799 let response = ClientControlResponse::SupervisorRescan { result };
5800 Ok(vec![control_response_body_frame(
5801 &frame,
5802 &response,
5803 "ClientControlResponse::SupervisorRescan",
5804 )?])
5805 }
5806
5807 async fn handle_supervisor_release_reserved(
5808 &self,
5809 frame: Frame,
5810 module_id: String,
5811 ) -> Result<Vec<Frame>, RouterError> {
5812 let Some(context) = self.rescan.clone() else {
5813 return Ok(vec![control_error_frame(
5814 &frame,
5815 "release_unavailable",
5816 "reserved-id release requires a daemon started with a reloadable config path",
5817 )?]);
5818 };
5819 let operation_lock = self.supervisor.operation_lock();
5820 let _operation_guard = operation_lock.lock().await;
5821 let loaded = match crate::daemon_config::load(&context.config_path) {
5822 Ok(Some(config)) => config,
5823 Ok(None) => {
5824 return Ok(vec![control_error_frame(
5825 &frame,
5826 "invalid_daemon_config",
5827 format!(
5828 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5829 context.config_path.display()
5830 ),
5831 )?])
5832 }
5833 Err(err) => {
5834 return Ok(vec![control_error_frame(
5835 &frame,
5836 "invalid_daemon_config",
5837 format!("unable to verify reserved-id release against daemon config: {err}"),
5838 )?])
5839 }
5840 };
5841 if loaded
5842 .modules
5843 .iter()
5844 .any(|configured| configured.module_id == module_id)
5845 {
5846 return Ok(vec![control_error_frame(
5847 &frame,
5848 "reserved_module_configured",
5849 format!(
5850 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5851 ),
5852 )?]);
5853 }
5854 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5855 return Ok(vec![control_error_frame(
5856 &frame,
5857 "reserved_gate_not_retained",
5858 format!(
5859 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5860 ),
5861 )?]);
5862 }
5863
5864 let response = ClientControlResponse::SupervisorAck {
5865 module_id,
5866 applied: true,
5867 };
5868 Ok(vec![control_response_body_frame(
5869 &frame,
5870 &response,
5871 "ClientControlResponse::SupervisorAck",
5872 )?])
5873 }
5874
5875 async fn reconcile_supervised_modules(
5884 &self,
5885 supervisor: &Supervisor,
5886 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5887 preview: bool,
5888 requester: Principal,
5889 ) -> Result<SupervisorRescanResult, String> {
5890 let mut current = BTreeMap::new();
5891 for module in self.supervisor.list() {
5892 let (spec, health) = module.configuration().map_err(|err| {
5893 format!(
5894 "failed to read configuration for module_id '{}': {err}",
5895 module.module_id()
5896 )
5897 })?;
5898 let enabled = module
5899 .status()
5900 .map_err(|err| {
5901 format!(
5902 "failed to read status for module_id '{}': {err}",
5903 module.module_id()
5904 )
5905 })?
5906 .enabled;
5907 current.insert(
5908 module.module_id().to_string(),
5909 (module, spec, health, enabled),
5910 );
5911 }
5912 let configured = configured_modules
5913 .into_iter()
5914 .map(|module| (module.module_id.clone(), module))
5915 .collect::<BTreeMap<_, _>>();
5916
5917 let added = configured
5918 .keys()
5919 .filter(|module_id| !current.contains_key(*module_id))
5920 .cloned()
5921 .collect::<Vec<_>>();
5922 let removed = current
5923 .keys()
5924 .filter(|module_id| !configured.contains_key(*module_id))
5925 .cloned()
5926 .collect::<Vec<_>>();
5927 let mut changed_pending_reload = Vec::new();
5928 let mut configuration_changes = BTreeSet::new();
5929 let mut enabled_changes = BTreeSet::new();
5930 let mut unchanged = 0_u32;
5931
5932 for (module_id, configured_module) in &configured {
5933 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5934 else {
5935 continue;
5936 };
5937 let launch_changed = *current_spec != configured_module.module_spec();
5941 let configuration_changed =
5942 launch_changed || *current_health != configured_module.health;
5943 let enabled_changed = *current_enabled != configured_module.enabled;
5944 if configuration_changed {
5945 configuration_changes.insert(module_id.clone());
5946 }
5947 if launch_changed {
5948 changed_pending_reload.push(module_id.clone());
5949 }
5950 if enabled_changed {
5951 enabled_changes.insert(module_id.clone());
5952 }
5953 if !configuration_changed && !enabled_changed {
5954 unchanged = unchanged.saturating_add(1);
5955 }
5956 }
5957
5958 if preview {
5964 return Ok(SupervisorRescanResult {
5965 added,
5966 removed,
5967 changed_pending_reload,
5968 enabled_changes: enabled_changes.iter().cloned().collect(),
5969 unchanged,
5970 preview: true,
5971 restart_required: Vec::new(),
5975 capability_warnings: Vec::new(),
5976 });
5977 }
5978
5979 for module_id in &removed {
5980 let module = ¤t
5981 .get(module_id)
5982 .expect("removed module came from current supervisor state")
5983 .0;
5984 module
5985 .retire_requested(Some(requester.clone()))
5986 .await
5987 .map_err(|err| {
5988 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5989 })?;
5990 self.supervisor.record_rescan_removal(module_id);
6024 self.supervisor.retire(module_id);
6025 self.route_outages.forget(module_id);
6026 }
6027
6028 for module_id in configured.keys() {
6029 let Some((module, _, _, _)) = current.get(module_id) else {
6030 continue;
6031 };
6032 let configured_module = configured
6033 .get(module_id)
6034 .expect("configured module id came from configured map");
6035 if configuration_changes.contains(module_id) {
6036 module
6037 .update_configuration(
6038 configured_module.module_spec(),
6039 configured_module.health.clone(),
6040 configured_module.drain_timeout_ms,
6041 )
6042 .await
6043 .map_err(|err| {
6044 format!(
6045 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6046 )
6047 })?;
6048 }
6049 if enabled_changes.contains(module_id) {
6050 self.route_outages.mark_operator_action(module_id);
6053 module
6054 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6055 .await
6056 .map_err(|err| {
6057 self.route_outages.operator_action_ended_unrefused(module_id);
6058 format!(
6059 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6060 configured_module.enabled
6061 )
6062 })?;
6063 }
6064 }
6065
6066 for module_id in &added {
6067 let configured_module = configured
6068 .get(module_id)
6069 .expect("added module id came from configured map");
6070 supervisor
6071 .supervise_configured_requested(
6072 configured_module.module_spec(),
6073 configured_module.enabled,
6074 configured_module.health.clone(),
6075 configured_module.drain_timeout_ms,
6076 configured_module.restart,
6077 Some(requester.clone()),
6078 )
6079 .map_err(|err| {
6080 format!("failed to add module_id '{module_id}' during rescan: {err}")
6081 })?;
6082 }
6083
6084 Ok(SupervisorRescanResult {
6085 added,
6086 removed,
6087 changed_pending_reload,
6088 enabled_changes: enabled_changes.iter().cloned().collect(),
6089 unchanged,
6090 preview: false,
6091 restart_required: Vec::new(),
6094 capability_warnings: Vec::new(),
6095 })
6096 }
6097
6098 async fn handle_supervisor_set_enabled(
6099 &self,
6100 frame: Frame,
6101 module_id: String,
6102 enabled: bool,
6103 requester: Principal,
6104 ) -> Result<Vec<Frame>, RouterError> {
6105 let operation_lock = self.supervisor.operation_lock();
6106 let _operation_guard = operation_lock.lock().await;
6107 let Some(module) = self.supervisor.get(&module_id) else {
6108 return Ok(vec![control_error_frame(
6109 &frame,
6110 "unknown_module",
6111 format!("module_id '{module_id}' is not supervised"),
6112 )?]);
6113 };
6114
6115 self.route_outages.mark_operator_action(&module_id);
6118 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6119 Ok(applied) => applied,
6120 Err(err) => {
6121 self.route_outages
6122 .operator_action_ended_unrefused(&module_id);
6123 return Ok(vec![control_error_frame(
6124 &frame,
6125 "target_unavailable",
6126 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6127 )?]);
6128 }
6129 };
6130 if !applied {
6131 self.route_outages
6134 .operator_action_ended_unrefused(&module_id);
6135 }
6136
6137 self.capability_evaluator.wake_deadline_loop();
6138 self.refresh_capability_requirements();
6139 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6140 Ok(vec![control_response_body_frame(
6141 &frame,
6142 &response,
6143 "ClientControlResponse::SupervisorAck",
6144 )?])
6145 }
6146
6147 async fn handle_supervisor_health_probe(
6148 &self,
6149 frame: Frame,
6150 module_id: String,
6151 ) -> Result<Vec<Frame>, RouterError> {
6152 self.refresh_capability_requirements();
6153 let Some(registration) = self
6154 .registry
6155 .get_module(&module_id)
6156 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6157 else {
6158 return Ok(vec![control_error_frame(
6159 &frame,
6160 "unknown_module",
6161 format!("module_id '{module_id}' is not registered"),
6162 )?]);
6163 };
6164
6165 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6178 {
6179 return Ok(vec![control_error_frame(
6180 &frame,
6181 "health_not_advertised",
6182 format!("module_id '{module_id}' did not advertise health.check"),
6183 )?]);
6184 }
6185
6186 let deadline = Instant::now() + self.health_probe_timeout;
6187 let pending = match self.forwarding.begin_module_control_rpc_for(
6188 &module_id,
6189 MODULE_CONTROL_OP_HEALTH_CHECK,
6190 deadline,
6191 ) {
6192 Ok(pending) => pending,
6193 Err(err) => {
6194 return Ok(vec![control_error_frame(
6195 &frame,
6196 forwarding_error_code(&err),
6197 err.to_string(),
6198 )?])
6199 }
6200 };
6201
6202 let PendingModuleControlRpc {
6203 endpoint,
6204 module_sink,
6205 negotiated_ver,
6206 corr: probe_corr,
6207 receiver,
6208 } = pending;
6209 let mut guard =
6210 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6211 let probe_body =
6212 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6213 RouterError::backend(
6214 0,
6215 frame.header.corr,
6216 format!("failed to encode health.check request: {err}"),
6217 )
6218 })?;
6219 let probe_frame = Frame::build_with_version(
6220 negotiated_ver,
6221 FrameType::Request,
6222 control_flags(),
6223 0,
6224 0,
6225 probe_corr,
6226 probe_body,
6227 )
6228 .map_err(RouterError::FrameBuild)?;
6229
6230 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6233 Ok(Ok(())) => {}
6234 Ok(Err(err)) => {
6235 return Ok(vec![control_error_frame(
6236 &frame,
6237 "target_unavailable",
6238 err.to_string(),
6239 )?]);
6240 }
6241 Err(_) => {
6242 return Ok(vec![control_error_frame(
6243 &frame,
6244 "module_timeout",
6245 format!(
6246 "module_id '{module_id}' did not answer health.check within {:?}",
6247 self.health_probe_timeout
6248 ),
6249 )?]);
6250 }
6251 }
6252
6253 match timeout_at(deadline, receiver).await {
6254 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6255 guard.disarm();
6256 let Some(report) = response.health_report() else {
6257 return Ok(vec![control_error_frame(
6258 &frame,
6259 "invalid_control_body",
6260 "health.check RPC returned a non-health response",
6261 )?]);
6262 };
6263 let HealthReport {
6268 status,
6269 detail,
6270 metrics,
6271 } = report;
6272 let capability_detail = self
6273 .capability_evaluator
6274 .required_problem_detail(&module_id);
6275 let response = ClientControlResponse::SupervisorHealthProbe {
6276 module_id,
6277 status,
6278 detail: append_capability_problem_detail(detail, capability_detail),
6279 metrics,
6280 };
6281 Ok(vec![control_response_body_frame(
6282 &frame,
6283 &response,
6284 "ClientControlResponse::SupervisorHealthProbe",
6285 )?])
6286 }
6287 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6288 guard.disarm();
6289 Ok(vec![control_error_body_frame(&frame, body)?])
6290 }
6291 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6292 guard.disarm();
6293 Ok(vec![control_error_frame(
6294 &frame,
6295 "target_unavailable",
6296 message,
6297 )?])
6298 }
6299 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6300 guard.disarm();
6301 Ok(vec![control_error_frame(
6302 &frame,
6303 "invalid_control_body",
6304 message,
6305 )?])
6306 }
6307 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6308 guard.disarm();
6309 Ok(vec![control_error_frame(
6310 &frame,
6311 "invalid_control_body",
6312 format!("expected module-control op '{expected}', got '{actual}'"),
6313 )?])
6314 }
6315 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6316 guard.disarm();
6317 Ok(vec![control_error_frame(
6318 &frame,
6319 "module_timeout",
6320 format!(
6321 "module_id '{module_id}' answered health.check after {:?}",
6322 self.health_probe_timeout
6323 ),
6324 )?])
6325 }
6326 Ok(Err(_)) => Ok(vec![control_error_frame(
6327 &frame,
6328 "target_unavailable",
6329 "health.check waiter was canceled before the module responded",
6330 )?]),
6331 Err(_) => Ok(vec![control_error_frame(
6332 &frame,
6333 "module_timeout",
6334 format!(
6335 "module_id '{module_id}' did not answer health.check within {:?}",
6336 self.health_probe_timeout
6337 ),
6338 )?]),
6339 }
6340 }
6341
6342 fn supervisor_status(
6343 &self,
6344 module_id: &str,
6345 corr: u64,
6346 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6347 self.supervisor
6348 .get(module_id)
6349 .map(|module| {
6350 let warming = module.is_warming_for_control("status").map_err(|err| {
6351 RouterError::backend(
6352 0,
6353 corr,
6354 format!(
6355 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6356 ),
6357 )
6358 })?;
6359 module.status_for_control("status").map_err(|err| {
6360 RouterError::backend(
6361 0,
6362 corr,
6363 format!(
6364 "failed to read supervisor status for module_id '{module_id}': {err}"
6365 ),
6366 )
6367 }).map(|status| (status, warming))
6368 })
6369 .transpose()
6370 }
6371
6372 fn guard_module_control_op(
6373 &self,
6374 frame: &Frame,
6375 module_id: &str,
6376 op: &str,
6377 ) -> Result<Option<Frame>, RouterError> {
6378 if self.module_grants_op(module_id, op, frame.header.corr)? {
6379 return Ok(None);
6380 }
6381
6382 Ok(Some(control_error_frame(
6383 frame,
6384 "op_not_allowed",
6385 format!("module_id '{module_id}' did not grant control op '{op}'"),
6386 )?))
6387 }
6388
6389 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6390 let Some(registration) = self
6391 .registry
6392 .get_module(module_id)
6393 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6394 else {
6395 return Ok(false);
6396 };
6397 Ok(module_registration_grants_op(®istration.control_ops, op))
6398 }
6399
6400 fn handle_status_update(
6401 &self,
6402 endpoint: ModuleEndpointId,
6403 frame: Frame,
6404 ) -> Result<Vec<Frame>, RouterError> {
6405 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6406 Ok(update) => update,
6407 Err(err) => {
6408 if is_known_module_push_op(&frame.body) {
6413 return Ok(vec![control_error_frame(
6414 &frame,
6415 "invalid_control_body",
6416 format!("malformed module control push body: {err}"),
6417 )?]);
6418 }
6419 return Ok(Vec::new());
6420 }
6421 };
6422
6423 match update {
6424 ModuleControlPush::RouteStatus {
6425 route_channel,
6426 route_epoch,
6427 status,
6428 } => {
6429 self.forwarding
6430 .cache_status(endpoint, route_channel, route_epoch, status)
6431 .map_err(RouterError::Forwarding)?;
6432 }
6433 }
6434 Ok(Vec::new())
6435 }
6436
6437 fn handle_route_poll(
6438 &self,
6439 ctx: &RouteCtx,
6440 frame: Frame,
6441 route_channel: u16,
6442 route_epoch: u32,
6443 kind: PollKind,
6444 ) -> Result<Vec<Frame>, RouterError> {
6445 let snapshot = self
6446 .forwarding
6447 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6448 .map_err(RouterError::Forwarding)?;
6449 let response = match (kind, snapshot) {
6450 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6451 ClientControlResponse::RoutePoll {
6452 route_channel,
6453 route_epoch,
6454 status,
6455 live: None,
6456 }
6457 }
6458 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6459 route_channel,
6460 route_epoch,
6461 status: None,
6462 live: None,
6463 },
6464 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6465 let live = self
6491 .process_liveness
6492 .as_ref()
6493 .and_then(|source| source.process_live(&module_id))
6494 .unwrap_or(true);
6495 ClientControlResponse::RoutePoll {
6496 route_channel,
6497 route_epoch,
6498 status: None,
6499 live: Some(live),
6500 }
6501 }
6502 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6503 route_channel,
6504 route_epoch,
6505 status: None,
6506 live: Some(false),
6507 },
6508 };
6509
6510 Ok(vec![control_response_body_frame(
6511 &frame,
6512 &response,
6513 "ClientControlResponse::RoutePoll",
6514 )?])
6515 }
6516
6517 pub(crate) fn observe_module_control_completion(
6518 &self,
6519 completion: ModuleControlRpcCompletion,
6520 ) -> bool {
6521 match completion {
6522 ModuleControlRpcCompletion::Unknown => false,
6523 ModuleControlRpcCompletion::Settled => true,
6524 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6525 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6526 info!(
6527 module_id = %module_id,
6528 latency_ms,
6529 "late health.check answer proves the module is alive"
6530 );
6531 match self
6532 .supervisor
6533 .record_late_health_answer(&module_id, latency_ms)
6534 {
6535 Ok(true) => {}
6536 Ok(false) => debug!(
6537 module_id = %module_id,
6538 latency_ms,
6539 "late health.check answer has no active supervisor snapshot"
6540 ),
6541 Err(err) => warn!(
6542 module_id = %module_id,
6543 latency_ms,
6544 error = %err,
6545 "failed to record late health.check answer"
6546 ),
6547 }
6548 true
6549 }
6550 }
6551 }
6552
6553 fn refuse_to_end_module_connection_for_a_client(
6574 &self,
6575 module_connection_id: ConnectionId,
6576 corr: u64,
6577 err: ForwardingError,
6578 ) -> Result<(), RouterError> {
6579 if let ForwardingError::ConnectionClosing { connection_id } = err {
6580 if connection_id != module_connection_id {
6581 warn!(
6582 module_connection_id = module_connection_id.get(),
6583 client_connection_id = connection_id.get(),
6584 corr,
6585 "dropping a route.bind response for a closing client; the module connection keeps serving"
6586 );
6587 return Ok(());
6588 }
6589 }
6590 Err(RouterError::Forwarding(err))
6591 }
6592
6593 fn handle_module_relay_response(
6594 &self,
6595 connection_id: ConnectionId,
6596 frame: Frame,
6597 ) -> Result<Vec<Frame>, RouterError> {
6598 let mut secondary_error = None;
6599 let outcome = match frame.header.ty {
6600 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6601 Ok(probe) if probe.op == "route.bind" => {
6602 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6603 Ok(ModuleControlResponse::RouteBindAck {}) => {
6604 RouteBindRelayOutcome::Accepted
6605 }
6606 Ok(other) => {
6607 let message =
6608 format!("route.bind response carried unexpected body: {other:?}");
6609 secondary_error = Some(control_error_frame(
6610 &frame,
6611 "invalid_control_body",
6612 message.clone(),
6613 )?);
6614 RouteBindRelayOutcome::ModuleGone(message)
6615 }
6616 Err(err) => {
6617 let message = format!("malformed route.bind response body: {err}");
6618 secondary_error = Some(control_error_frame(
6619 &frame,
6620 "invalid_control_body",
6621 message.clone(),
6622 )?);
6623 RouteBindRelayOutcome::ModuleGone(message)
6624 }
6625 }
6626 }
6627 Ok(probe) => {
6628 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6629 {
6630 Ok(response) => ModuleControlRpcOutcome::Response(response),
6631 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6632 "malformed {} response body: {err}",
6633 probe.op
6634 )),
6635 };
6636 let completion = self
6637 .forwarding
6638 .complete_module_control_rpc(
6639 connection_id,
6640 frame.header.corr,
6641 Some(&probe.op),
6642 outcome,
6643 )
6644 .map_err(RouterError::Forwarding)?;
6645 if !self.observe_module_control_completion(completion) {
6646 debug!(
6647 connection_id = connection_id.get(),
6648 corr = frame.header.corr,
6649 op = %probe.op,
6650 "dropping late or unknown module-control RPC response"
6651 );
6652 }
6653 return Ok(Vec::new());
6654 }
6655 Err(err) => {
6656 if let Some(expected_op) = self
6657 .forwarding
6658 .pending_module_control_op(connection_id, frame.header.corr)
6659 .map_err(RouterError::Forwarding)?
6660 {
6661 let completion = self
6662 .forwarding
6663 .complete_module_control_rpc(
6664 connection_id,
6665 frame.header.corr,
6666 None,
6667 ModuleControlRpcOutcome::MalformedResponse(format!(
6668 "malformed {expected_op} response body: {err}"
6669 )),
6670 )
6671 .map_err(RouterError::Forwarding)?;
6672 if !self.observe_module_control_completion(completion) {
6673 debug!(
6674 connection_id = connection_id.get(),
6675 corr = frame.header.corr,
6676 "dropping late malformed module-control RPC response"
6677 );
6678 }
6679 return Ok(Vec::new());
6680 }
6681 let message = format!("malformed route.bind response body: {err}");
6682 secondary_error = Some(control_error_frame(
6683 &frame,
6684 "invalid_control_body",
6685 message.clone(),
6686 )?);
6687 RouteBindRelayOutcome::ModuleGone(message)
6688 }
6689 },
6690 FrameType::Error => {
6691 if self
6692 .forwarding
6693 .pending_module_control_op(connection_id, frame.header.corr)
6694 .map_err(RouterError::Forwarding)?
6695 .is_some()
6696 {
6697 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6698 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6699 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6700 "malformed module-control ERROR body: {err}"
6701 )),
6702 };
6703 let completion = self
6704 .forwarding
6705 .complete_module_control_rpc(
6706 connection_id,
6707 frame.header.corr,
6708 None,
6709 outcome,
6710 )
6711 .map_err(RouterError::Forwarding)?;
6712 if !self.observe_module_control_completion(completion) {
6713 debug!(
6714 connection_id = connection_id.get(),
6715 corr = frame.header.corr,
6716 "dropping late or unknown module-control RPC error"
6717 );
6718 }
6719 return Ok(Vec::new());
6720 }
6721 match serde_json::from_slice::<ErrorBody>(&frame.body) {
6722 Ok(body) => RouteBindRelayOutcome::Rejected(body),
6723 Err(err) => {
6724 let message = format!("malformed route.bind ERROR body: {err}");
6725 secondary_error = Some(control_error_frame(
6726 &frame,
6727 "invalid_control_body",
6728 message.clone(),
6729 )?);
6730 RouteBindRelayOutcome::ModuleGone(message)
6731 }
6732 }
6733 }
6734 ty => {
6735 return Ok(vec![control_error_frame(
6736 &frame,
6737 "unsupported_control_frame",
6738 format!("unsupported module channel-0 frame {ty:?}"),
6739 )?])
6740 }
6741 };
6742
6743 let settled =
6744 self.forwarding
6745 .complete_pending_relay(connection_id, frame.header.corr, outcome);
6746 let completion = match settled {
6747 Ok(completion) => completion,
6748 Err(err) => {
6749 self.refuse_to_end_module_connection_for_a_client(
6750 connection_id,
6751 frame.header.corr,
6752 err,
6753 )?;
6754 return Ok(secondary_error.into_iter().collect());
6755 }
6756 };
6757 if let Some(target) = completion.abandoned.as_ref() {
6758 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6759 }
6760 if !completion.settled {
6761 debug!(
6762 connection_id = connection_id.get(),
6763 corr = frame.header.corr,
6764 frame_type = ?frame.header.ty,
6765 "dropping late or unknown route.bind relay response"
6766 );
6767 }
6768 Ok(secondary_error.into_iter().collect())
6769 }
6770
6771 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6772 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6773 self.cleanup_connection_with_end_reason(
6777 connection_id,
6778 RegistrationEndReason::ExplicitGoodbye,
6779 )
6780 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6781 Ok(Vec::new())
6782 }
6783}
6784
6785impl Default for ControlHandler {
6786 fn default() -> Self {
6787 Self::new(Arc::new(Registry::default()))
6788 }
6789}
6790
6791impl crate::supervise::SwapPromotionObserver for ControlHandler {
6792 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6793 self.apply_registration_capabilities(registration);
6794 }
6795}
6796
6797fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6798 CapabilityRequirementStatus {
6799 consumer: status.consumer,
6800 capability: status.capability,
6801 need: match status.need {
6802 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6803 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6804 },
6805 verdict: status.verdict.as_str().to_string(),
6806 episode_seq: status.episode_seq,
6807 config_satisfiable: status.config_satisfiable,
6808 runtime_available: status.runtime_available,
6809 detail: status.detail,
6810 }
6811}
6812
6813fn append_capability_problem_detail(
6814 detail: Option<String>,
6815 capability_detail: Option<String>,
6816) -> Option<String> {
6817 match (detail, capability_detail) {
6818 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6819 (Some(detail), None) => Some(detail),
6820 (None, Some(capability_detail)) => Some(capability_detail),
6821 (None, None) => None,
6822 }
6823}
6824
6825fn subc_ops() -> Vec<String> {
6826 SUBC_CONTROL_OPS
6827 .iter()
6828 .map(|op| (*op).to_string())
6829 .collect()
6830}
6831
6832fn module_subc_ops() -> Vec<String> {
6833 SUBC_CONTROL_OPS
6834 .iter()
6835 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6836 .map(|op| (*op).to_string())
6837 .collect()
6838}
6839
6840#[cfg(test)]
6841fn module_baseline_control_ops() -> Vec<String> {
6842 MODULE_BASELINE_CONTROL_OPS
6843 .iter()
6844 .map(|op| (*op).to_string())
6845 .collect()
6846}
6847
6848fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6849 let mut seen = HashSet::new();
6850 let mut effective = Vec::new();
6851 for op in MODULE_BASELINE_CONTROL_OPS {
6852 if seen.insert((*op).to_string()) {
6853 effective.push((*op).to_string());
6854 }
6855 }
6856 for op in declared.unwrap_or_default() {
6857 if seen.insert(op.clone()) {
6858 effective.push(op);
6859 }
6860 }
6861 effective
6862}
6863
6864fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6865 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6866}
6867
6868fn target_module_id(target: &RouteTarget) -> &str {
6869 match target {
6870 RouteTarget::ToolProvider { module_id }
6871 | RouteTarget::ManagementSurface { module_id }
6872 | RouteTarget::InternalService { module_id, .. } => module_id,
6873 }
6874}
6875
6876fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6877 roles.iter().any(|role| match (target, role) {
6878 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6879 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6880 (
6881 RouteTarget::InternalService { service_id, .. },
6882 ProviderRole::InternalService {
6883 service_id: provided,
6884 ..
6885 },
6886 ) => service_id == provided,
6887 _ => false,
6888 })
6889}
6890
6891fn is_routable_role(role: &ProviderRole) -> bool {
6892 matches!(
6893 role,
6894 ProviderRole::ToolProvider { .. }
6895 | ProviderRole::ManagementSurface { .. }
6896 | ProviderRole::InternalService { .. }
6897 )
6898}
6899
6900#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6901enum ControlRequestBodyError {
6902 UnknownOp,
6903 InvalidBody,
6904}
6905
6906#[derive(Debug, Deserialize)]
6907struct ControlOpProbe {
6908 op: String,
6909}
6910
6911const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6914
6915fn is_known_module_push_op(body: &[u8]) -> bool {
6916 serde_json::from_slice::<ControlOpProbe>(body)
6917 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6918 .unwrap_or(false)
6919}
6920
6921fn is_known_module_request_op(body: &[u8]) -> bool {
6922 serde_json::from_slice::<ControlOpProbe>(body)
6923 .map(|probe| is_module_to_subc_op(&probe.op))
6924 .unwrap_or(false)
6925}
6926
6927fn is_module_to_subc_op(op: &str) -> bool {
6928 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6929}
6930
6931fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6932 debug!(
6933 op = %op,
6934 connection_id = connection_id.get(),
6935 corr,
6936 "control dispatch"
6937 );
6938}
6939
6940fn log_slow_control_dispatch(
6941 dispatch_started_at: Option<StdInstant>,
6942 op: &'static str,
6943 connection_id: ConnectionId,
6944 corr: u64,
6945 route_open_timings: Option<&RouteOpenTimings>,
6946) {
6947 let Some(dispatch_started_at) = dispatch_started_at else {
6948 return;
6949 };
6950 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
6952 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6953 if let Some(timings) = route_open_timings {
6954 warn!(
6955 op = %op,
6956 connection_id = connection_id.get(),
6957 corr,
6958 elapsed_ms = elapsed.as_millis() as u64,
6959 module_id = ?timings.module_id,
6960 principal = %timings.principal,
6961 admission_ms = timings.admission.as_millis() as u64,
6962 project_root_ms = timings.project_root.as_millis() as u64,
6963 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
6964 bind_send_ms = timings.bind_send.as_millis() as u64,
6965 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
6966 "slow control dispatch"
6967 );
6968 return;
6969 }
6970 warn!(
6971 op = %op,
6972 connection_id = connection_id.get(),
6973 corr,
6974 elapsed_ms = elapsed.as_millis() as u64,
6975 "slow control dispatch"
6976 );
6977 }
6978}
6979
6980#[cfg(test)]
6982type TestCallerLookup =
6983 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
6984
6985fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
6987 match request {
6988 ClientControlRequest::SupervisorRestart { module_id, .. } => {
6989 Some(PrivilegedOperation::Restart { module_id })
6990 }
6991 ClientControlRequest::SupervisorSwap { module_id, .. } => {
6992 Some(PrivilegedOperation::Swap { module_id })
6993 }
6994 ClientControlRequest::SupervisorReload { module_id } => {
6995 Some(PrivilegedOperation::Reload { module_id })
6996 }
6997 ClientControlRequest::SupervisorRescan { preview } => {
6998 if *preview {
6999 None
7000 } else {
7001 Some(PrivilegedOperation::Rescan)
7002 }
7003 }
7004 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7005 Some(PrivilegedOperation::SetEnabled {
7006 module_id,
7007 enabled: *enabled,
7008 })
7009 }
7010 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7011 Some(PrivilegedOperation::ReleaseReserved { module_id })
7012 }
7013 ClientControlRequest::ServerDescribe {}
7014 | ClientControlRequest::CatalogList { .. }
7015 | ClientControlRequest::RouteOpen { .. }
7016 | ClientControlRequest::RoutePoll { .. }
7017 | ClientControlRequest::SupervisorList {}
7018 | ClientControlRequest::SupervisorHealth {}
7019 | ClientControlRequest::SupervisorHealthProbe { .. }
7020 | ClientControlRequest::SupervisorRoutes { .. }
7021 | ClientControlRequest::SupervisorProvenance { .. }
7022 | ClientControlRequest::SupervisorStderrTail { .. }
7023 | ClientControlRequest::SupervisorTerminals { .. }
7024 | ClientControlRequest::SupervisorSpawnSnapshot {}
7025 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7026 }
7027}
7028
7029fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7030 match request {
7031 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7032 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7033 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7034 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7035 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7036 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7037 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7038 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7039 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7040 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7041 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7042 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7043 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7044 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7045 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7046 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7047 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7048 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7049 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7050 }
7051}
7052
7053fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7059 use subc_protocol::scope::ScopeKind;
7060 const MAX_REF_CHARS: usize = 64;
7061 let kind = match stamp.kind {
7062 ScopeKind::Head => "head",
7063 ScopeKind::Worker => "worker",
7064 ScopeKind::Ephemeral => "ephemeral",
7065 };
7066 let mut scope_ref: String = stamp.scope_ref.chars().take(MAX_REF_CHARS).collect();
7067 if stamp.scope_ref.chars().nth(MAX_REF_CHARS).is_some() {
7068 scope_ref.push('…');
7069 }
7070 format!(
7071 "{kind}/{}/{scope_ref:?}@{}",
7072 principal_label(&stamp.owner),
7073 stamp.scope_epoch
7074 )
7075}
7076
7077fn principal_label(principal: &Principal) -> String {
7078 match principal {
7079 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7080 Principal::Direct => "direct".to_string(),
7081 other => format!("{other:?}"),
7082 }
7083}
7084
7085fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7086 match request {
7087 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7088 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7089 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7090 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7091 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7092 }
7093}
7094
7095fn parse_client_control_request(
7096 body: &[u8],
7097) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7098 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7099 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7100 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
7101 ControlRequestBodyError::InvalidBody
7102 }
7103 Ok(_) => ControlRequestBodyError::UnknownOp,
7104 Err(_) => ControlRequestBodyError::InvalidBody,
7105 };
7106 (err, classification)
7107 })
7108}
7109
7110fn parse_module_control_request_from_module(
7111 body: &[u8],
7112) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7113 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7114 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7115 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7116 Ok(_) => ControlRequestBodyError::UnknownOp,
7117 Err(_) => ControlRequestBodyError::InvalidBody,
7118 };
7119 (err, classification)
7120 })
7121}
7122
7123#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7124enum ProviderRoleKind {
7125 ToolProvider,
7126 PipelineStage,
7127 ManagementSurface,
7128 InternalService,
7129}
7130
7131fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7132 match role {
7133 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7134 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7135 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7136 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7137 }
7138}
7139
7140fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7141 roles.iter().map(provider_role_kind).collect()
7142}
7143
7144const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7147
7148#[derive(Debug, Default, PartialEq, Eq)]
7150struct ScopeOutcomeCounts {
7151 created: usize,
7152 replaced: usize,
7153 updated: usize,
7154 unchanged: usize,
7155 refused: usize,
7156}
7157
7158impl ScopeOutcomeCounts {
7159 fn of(results: &[ScopeRecordResult]) -> Self {
7160 let mut counts = Self::default();
7161 for result in results {
7162 let slot = match result.outcome {
7163 ScopeRecordOutcome::Created => &mut counts.created,
7164 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7165 ScopeRecordOutcome::Updated => &mut counts.updated,
7166 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7167 ScopeRecordOutcome::Refused => &mut counts.refused,
7168 };
7169 *slot += 1;
7170 }
7171 counts
7172 }
7173}
7174
7175#[cfg(test)]
7176mod scope_outcome_count_tests {
7177 use super::*;
7178
7179 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7180 ScopeRecordResult {
7181 scope_ref: "r".to_string(),
7182 scope_epoch: 1,
7183 outcome,
7184 code: None,
7185 message: None,
7186 version: None,
7187 parent_state: None,
7188 }
7189 }
7190
7191 #[test]
7194 fn every_outcome_is_counted_in_its_own_field() {
7195 let results = [
7196 result(ScopeRecordOutcome::Created),
7197 result(ScopeRecordOutcome::Created),
7198 result(ScopeRecordOutcome::Replaced),
7199 result(ScopeRecordOutcome::Updated),
7200 result(ScopeRecordOutcome::Unchanged),
7201 result(ScopeRecordOutcome::Refused),
7202 result(ScopeRecordOutcome::Refused),
7203 result(ScopeRecordOutcome::Refused),
7204 ];
7205 assert_eq!(
7206 ScopeOutcomeCounts::of(&results),
7207 ScopeOutcomeCounts {
7208 created: 2,
7209 replaced: 1,
7210 updated: 1,
7211 unchanged: 1,
7212 refused: 3,
7213 }
7214 );
7215 }
7216}
7217
7218fn capability_census_trigger(
7222 old: Option<&CapabilityDeclarations>,
7223 new: Option<&CapabilityDeclarations>,
7224) -> bool {
7225 let old_provides = old
7226 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7227 .unwrap_or_default();
7228 let old_denies = old
7229 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7230 .unwrap_or_default();
7231 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7232 provides: Vec::new(),
7233 requires: Vec::new(),
7234 must_never_reach: Vec::new(),
7235 });
7236
7237 new.provides
7238 .iter()
7239 .any(|capability| !old_provides.contains(capability))
7240 || new
7241 .must_never_reach
7242 .iter()
7243 .any(|capability| !old_denies.contains(capability))
7244}
7245
7246fn denied_capability<'a>(
7249 opening_manifest: &'a ModuleManifest,
7250 target_manifest: &ModuleManifest,
7251) -> Option<&'a str> {
7252 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7253 let target_capabilities = target_manifest.capabilities.as_ref()?;
7254 opening_capabilities
7255 .must_never_reach
7256 .iter()
7257 .find(|denied| {
7258 target_capabilities
7259 .provides
7260 .iter()
7261 .any(|provided| provided == *denied)
7262 })
7263 .map(String::as_str)
7264}
7265
7266fn catalog_update_frozen_field_message(
7267 registered: &ModuleManifest,
7268 provides: &[ProviderRole],
7269) -> Option<String> {
7270 let old_has_provides = !registered.provides.is_empty();
7271 let new_has_provides = !provides.is_empty();
7272 if old_has_provides != new_has_provides {
7273 return Some(format!(
7274 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7275 registered.module_id
7276 ));
7277 }
7278
7279 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7280 return Some(format!(
7281 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7282 registered.module_id
7283 ));
7284 }
7285
7286 let registered_concurrency = manifest_concurrency(registered);
7287 let mut candidate = registered.clone();
7288 candidate.provides = provides.to_vec();
7289 let candidate_concurrency = manifest_concurrency(&candidate);
7290 if candidate_concurrency != registered_concurrency {
7291 return Some(format!(
7292 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7293 registered.module_id, registered_concurrency, candidate_concurrency
7294 ));
7295 }
7296
7297 None
7300}
7301
7302fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7303 manifest.provides.iter().any(is_routable_role)
7304}
7305
7306fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7312 manifest
7313 .provides
7314 .iter()
7315 .find_map(|provider| match provider {
7316 ProviderRole::ToolProvider { concurrency, .. }
7317 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7318 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7319 })
7320 .unwrap_or(Concurrency::ModuleManaged)
7321}
7322
7323fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7330 let has_management_surface = manifest
7331 .provides
7332 .iter()
7333 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7334 if !has_management_surface {
7335 return false;
7336 }
7337 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7338 return false;
7339 };
7340 let Some(provides) = raw
7341 .get("manifest")
7342 .and_then(|manifest| manifest.get("provides"))
7343 .and_then(serde_json::Value::as_array)
7344 else {
7345 return false;
7346 };
7347 provides.iter().any(|role| {
7352 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7353 && role.get("concurrency").is_none()
7354 })
7355}
7356
7357fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7358 if peer_version != PROTOCOL_VERSION {
7359 return Err(format!(
7360 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7361 ));
7362 }
7363 Ok(PROTOCOL_VERSION)
7364}
7365
7366fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7367 Frame::build_with_version(
7368 response_version(frame),
7369 FrameType::Pong,
7370 frame.header.flags,
7371 0,
7372 0,
7373 frame.header.corr,
7374 Vec::new(),
7375 )
7376 .map_err(RouterError::FrameBuild)
7377}
7378
7379pub(crate) fn control_error_frame(
7380 frame: &Frame,
7381 code: &'static str,
7382 message: impl Into<String>,
7383) -> Result<Frame, RouterError> {
7384 control_error_body_frame(
7385 frame,
7386 ErrorBody {
7387 code: code.to_string(),
7388 message: message.into(),
7389 detail: None,
7390 },
7391 )
7392}
7393
7394fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7395 let body = serde_json::to_vec(&error).map_err(|err| {
7396 RouterError::backend(
7397 0,
7398 frame.header.corr,
7399 format!("failed to encode control ERROR: {err}"),
7400 )
7401 })?;
7402
7403 Frame::build_with_version(
7404 response_version(frame),
7405 FrameType::Error,
7406 control_flags(),
7407 0,
7408 0,
7409 frame.header.corr,
7410 body,
7411 )
7412 .map_err(RouterError::FrameBuild)
7413}
7414
7415fn control_response_body_frame<T: Serialize>(
7416 frame: &Frame,
7417 reply: &T,
7418 label: &'static str,
7419) -> Result<Frame, RouterError> {
7420 let body = serde_json::to_vec(reply).map_err(|err| {
7421 RouterError::backend(
7422 0,
7423 frame.header.corr,
7424 format!("failed to encode {label}: {err}"),
7425 )
7426 })?;
7427
7428 Frame::build_with_version(
7429 response_version(frame),
7430 FrameType::Response,
7431 control_flags(),
7432 0,
7433 0,
7434 frame.header.corr,
7435 body,
7436 )
7437 .map_err(RouterError::FrameBuild)
7438}
7439
7440fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7470 match err {
7471 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7472 ForwardingError::NoModuleConnection => "target_unavailable",
7473 ForwardingError::ModuleReloading { .. } => "module_reloading",
7474 ForwardingError::ClientRouteChannelExhausted { .. }
7475 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7476 ForwardingError::StaleModuleEndpoint
7477 | ForwardingError::UnknownReservation { .. }
7478 | ForwardingError::ConnectionClosing { .. }
7479 | ForwardingError::ClientEgressClosed { .. }
7480 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7481 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7484 ForwardingError::RelayCorrelationExhausted
7485 | ForwardingError::RouteOpenBuild(_)
7486 | ForwardingError::Poisoned => "forwarding_error",
7487 }
7488}
7489
7490fn response_version(frame: &Frame) -> u8 {
7491 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7492 frame.header.ver
7493 } else {
7494 PROTOCOL_VERSION
7495 }
7496}
7497
7498fn control_flags() -> Flags {
7499 Flags::new(false, Priority::Passive, false)
7500}
7501
7502fn send_goodbye_target_best_effort(
7507 counters: &DaemonCounters,
7508 target: &GoodbyeTarget,
7509 context: &'static str,
7510) {
7511 let Ok(frame) = Frame::build_with_version(
7512 target.negotiated_ver,
7513 FrameType::Goodbye,
7514 control_flags(),
7515 target.channel,
7516 target.epoch,
7517 0,
7518 Vec::new(),
7519 ) else {
7520 return;
7521 };
7522 crate::forwarding::send_module_route_goodbye(
7523 counters,
7524 &target.sink,
7525 frame,
7526 target.module_id.as_deref(),
7527 context,
7528 );
7529}
7530
7531pub(crate) fn send_route_control_pushes(
7532 forwarding: &ForwardingTable,
7533 routes: Vec<EndpointRoute>,
7534 push: ClientControlPush,
7535) {
7536 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7537 for route in routes {
7538 let target = route.goodbye_target;
7539 if let Some((existing, channels)) = targets
7540 .iter_mut()
7541 .find(|(existing, _)| existing.connection_id == target.connection_id)
7542 {
7543 debug_assert_eq!(
7544 existing.negotiated_ver, target.negotiated_ver,
7545 "one connection cannot negotiate multiple frame versions"
7546 );
7547 if !channels.contains(&target.channel) {
7548 channels.push(target.channel);
7549 }
7550 continue;
7551 }
7552 let channel = target.channel;
7553 targets.push((target, vec![channel]));
7554 }
7555 for (target, mut channels) in targets {
7556 channels.sort_unstable();
7557 let mut push = push.clone();
7558 match &mut push {
7559 ClientControlPush::RouteClosing {
7560 channels: covered, ..
7561 }
7562 | ClientControlPush::RouteClosed {
7563 channels: covered, ..
7564 } => *covered = channels,
7565 }
7566 let body = match serde_json::to_vec(&push) {
7567 Ok(body) => body,
7568 Err(err) => {
7569 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7570 continue;
7571 }
7572 };
7573 let frame = match Frame::build_with_version(
7574 target.negotiated_ver,
7575 FrameType::Push,
7576 control_flags(),
7577 0,
7578 0,
7579 0,
7580 body.clone(),
7581 ) {
7582 Ok(frame) => frame,
7583 Err(err) => {
7584 warn!(
7585 route_channel = target.channel,
7586 error = %err,
7587 "failed to build route lifecycle control PUSH frame"
7588 );
7589 continue;
7590 }
7591 };
7592 if let Err(err) = target.sink.try_send(frame) {
7593 if target.close_on_delivery_failure() {
7594 warn!(
7595 target_connection_id = target.connection_id.get(),
7596 route_channel = target.channel,
7597 error = %err,
7598 "route lifecycle control PUSH was not delivered to client; closing target connection"
7599 );
7600 let _ = forwarding.escalate_client_delivery_failure(
7601 target.connection_id,
7602 target.channel,
7603 target.epoch,
7604 CloseReason::new(
7605 "route_lifecycle_push_delivery_failed",
7606 format!(
7607 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7608 target.channel
7609 ),
7610 ),
7611 crate::forwarding::UndeliveredFrame {
7612 module_id: target.module_id.as_deref(),
7613 sink: &target.sink,
7614 },
7615 );
7616 }
7617 }
7618 }
7619}
7620
7621#[cfg(test)]
7622mod tests {
7623 mod activity_read {
7624 include!("activity_read_tests.rs");
7625 }
7626
7627 mod lifecycle_activity {
7628 include!("control_lifecycle_tests.rs");
7629 }
7630 #[cfg(unix)]
7631 #[tokio::test]
7632 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7633 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7634 let path = dir.join("subc.jsonc");
7635 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7636 "program":"/bin/sleep","args":["60"],"protocol":"none",
7637 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7638 }}}).to_string()).unwrap();
7639 let mut configured = crate::daemon_config::load(&path)
7640 .unwrap()
7641 .unwrap()
7642 .modules
7643 .pop()
7644 .unwrap();
7645 let registry = std::sync::Arc::new(crate::Registry::default());
7646 let handle = crate::SupervisorHandle::new();
7647 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7648 .with_handle(handle.clone());
7649 let module = supervisor
7650 .supervise_configured_with_health(
7651 configured.module_spec(),
7652 true,
7653 configured.health.clone(),
7654 None,
7655 configured.restart,
7656 )
7657 .unwrap();
7658 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7659 let before = module.status().unwrap().pid;
7660 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7661 configured.health.cadence = std::time::Duration::from_secs(3600);
7662 let health_only = handler
7663 .reconcile_supervised_modules(
7664 &supervisor,
7665 vec![configured.clone()],
7666 false,
7667 Principal::Direct,
7668 )
7669 .await
7670 .unwrap();
7671 assert!(
7672 health_only.changed_pending_reload.is_empty(),
7673 "health policy is already applied live"
7674 );
7675 assert_eq!(module.status().unwrap().pid, before);
7676 assert_eq!(
7677 module.configuration().unwrap().1.http,
7678 configured.health.http
7679 );
7680 configured.args = vec!["61".into()];
7681 let launch = handler
7682 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7683 .await
7684 .unwrap();
7685 assert_eq!(launch.changed_pending_reload, ["stock"]);
7686 assert_eq!(
7687 module.status().unwrap().pid,
7688 before,
7689 "a launch edit is stored until reload"
7690 );
7691 module.drain().await.unwrap();
7692 }
7693 use cortexkit_test_support::ScratchDir;
7694 use std::{
7695 collections::BTreeMap,
7696 fmt,
7697 path::PathBuf,
7698 sync::{Arc, Mutex},
7699 time::Duration,
7700 };
7701
7702 use serde_json::{json, Value};
7703 use subc_protocol::{
7704 manifest::{
7705 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7706 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7707 },
7708 session::HealthStatus,
7709 FrameType,
7710 };
7711
7712 use super::*;
7713 use crate::{
7714 forwarding::{DataRoute, DataRouteState},
7715 registry::ChannelState,
7716 router::FrameSink,
7717 stderr_tail::DEFAULT_MAX_LINE_BYTES,
7718 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7719 RouteCtx, Router,
7720 };
7721 use tokio::{
7722 sync::mpsc,
7723 time::{sleep, Instant},
7724 };
7725 use tracing::{
7726 field::{Field, Visit},
7727 Event, Subscriber,
7728 };
7729 use tracing_subscriber::{layer::Context, prelude::*, Layer};
7730
7731 fn fake_aft_stub_path() -> PathBuf {
7750 let mut path = std::env::current_exe().expect("current_exe available in tests");
7751 path.pop(); path.pop(); path.push(if cfg!(windows) {
7754 "fake-aft-stub.exe"
7755 } else {
7756 "fake-aft-stub"
7757 });
7758 assert!(
7759 path.exists(),
7760 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7761 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7762 path.display()
7763 );
7764 path
7765 }
7766
7767 fn client_retries(code: &str) -> bool {
7772 subc_protocol::error_codes::is_retryable_route_open(code)
7773 }
7774
7775 #[test]
7786 fn retryability_of_forwarding_codes_matches_the_failure() {
7787 let transient = [
7790 ForwardingError::NoModuleConnection,
7791 ForwardingError::ModuleReloading {
7792 module_id: "m".into(),
7793 },
7794 ForwardingError::StaleModuleEndpoint,
7795 ForwardingError::UnknownReservation {
7796 client_channel: 1,
7797 module_channel: 1,
7798 },
7799 ForwardingError::ConnectionClosing {
7800 connection_id: ConnectionId::new(1),
7801 },
7802 ForwardingError::ClientEgressClosed {
7803 connection_id: ConnectionId::new(1),
7804 },
7805 ForwardingError::ModuleEgressUnavailable {
7806 connection_id: ConnectionId::new(1),
7807 },
7808 ];
7809 for err in transient {
7810 let code = forwarding_error_code(&err);
7811 assert!(
7812 client_retries(code),
7813 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7814 );
7815 }
7816
7817 let permanent = [
7822 ForwardingError::ConnectionRoleConflict {
7823 connection_id: ConnectionId::new(1),
7824 },
7825 ForwardingError::ClientRouteChannelExhausted {
7826 connection_id: ConnectionId::new(1),
7827 },
7828 ForwardingError::ModuleRouteChannelExhausted {
7829 endpoint: ModuleEndpointId {
7830 connection_id: ConnectionId::new(1),
7831 generation: 1,
7832 },
7833 },
7834 ForwardingError::RelayCorrelationExhausted,
7835 ForwardingError::RouteOpenBuild("x".into()),
7836 ForwardingError::Poisoned,
7837 ];
7838 for err in permanent {
7839 let code = forwarding_error_code(&err);
7840 assert!(
7841 !client_retries(code),
7842 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7843 );
7844 }
7845 }
7846
7847 #[tokio::test]
7875 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7876 let handler = ControlHandler::default();
7877 let frame =
7878 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7879
7880 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7884 assert_eq!(
7885 stamped,
7886 Principal::Direct,
7887 "a caller that proved nothing must not be stamped as a supervised module"
7888 );
7889
7890 let forged = handler
7895 .route_open_principal(
7896 &frame,
7897 Some(ConsumerIdentity {
7898 module_id: "aft".to_string(),
7899 launch_nonce: "not-a-real-nonce".to_string(),
7900 }),
7901 )
7902 .unwrap();
7903 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7904 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7905 }
7906
7907 #[test]
7924 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7925 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7926 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7927 let ClientControlRequest::RouteOpen {
7928 consumer_identity, ..
7929 } = parsed
7930 else {
7931 panic!("route.open body must parse as RouteOpen");
7932 };
7933 assert_eq!(
7934 consumer_identity,
7935 Some(ConsumerIdentity {
7936 module_id: "aft".to_string(),
7937 launch_nonce: "n".to_string(),
7938 }),
7939 "the wire field name must reach the value route_open_principal reads"
7940 );
7941 }
7942
7943 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7944 ModuleManifest::builder(module_id, "0.1.0")
7945 .protocol_ver(protocol_ver)
7946 .provides(vec![ProviderRole::ToolProvider {
7947 tools: vec![Tool {
7948 name: "read".to_string(),
7949 description: None,
7950 execution_mode: ExecutionMode::Pure,
7951 schema: json!({"type": "object"}),
7952 }],
7953 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7954 concurrency: Concurrency::ModuleManaged,
7955 emits_push: true,
7956 sub_supervises: true,
7957 }])
7958 .build()
7959 }
7960
7961 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7962 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7963 }
7964
7965 fn hello_frame_with_control_ops(
7966 module_id: &str,
7967 protocol_ver: u8,
7968 corr: u64,
7969 control_ops: Option<Vec<String>>,
7970 ) -> Frame {
7971 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7972 }
7973
7974 fn hello_frame_with_nonce(
7975 module_id: &str,
7976 protocol_ver: u8,
7977 corr: u64,
7978 launch_nonce: Option<&str>,
7979 ) -> Frame {
7980 hello_frame_full(
7981 module_id,
7982 protocol_ver,
7983 corr,
7984 None,
7985 launch_nonce.map(ToOwned::to_owned),
7986 )
7987 }
7988
7989 fn hello_frame_full(
7990 module_id: &str,
7991 protocol_ver: u8,
7992 corr: u64,
7993 control_ops: Option<Vec<String>>,
7994 launch_nonce: Option<String>,
7995 ) -> Frame {
7996 let body = serde_json::to_vec(&ModuleHelloBody {
7997 manifest: manifest(module_id, protocol_ver),
7998 protocol_ver,
7999 control_ops,
8000 launch_nonce,
8001 })
8002 .unwrap();
8003 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8004 }
8005
8006 fn non_routable_hello_frame_with_control_ops(
8007 module_id: &str,
8008 corr: u64,
8009 control_ops: Option<Vec<String>>,
8010 ) -> Frame {
8011 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8012 manifest.provides.clear();
8013 let body = serde_json::to_vec(&ModuleHelloBody {
8014 manifest,
8015 protocol_ver: PROTOCOL_VERSION,
8016 control_ops,
8017 launch_nonce: None,
8018 })
8019 .unwrap();
8020 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8021 }
8022
8023 fn capability_grammar_hello_frame(
8024 capabilities: Value,
8025 runtime_computed: Option<Value>,
8026 corr: u64,
8027 ) -> Frame {
8028 let mut body = serde_json::to_value(ModuleHelloBody {
8029 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8030 protocol_ver: PROTOCOL_VERSION,
8031 control_ops: None,
8032 launch_nonce: None,
8033 })
8034 .expect("HELLO body serializes");
8035 body["manifest"]["capabilities"] = capabilities;
8036 if let Some(runtime_computed) = runtime_computed {
8037 body["runtime_computed"] = runtime_computed;
8038 }
8039 Frame::build(
8040 FrameType::Hello,
8041 control_flags(),
8042 0,
8043 0,
8044 corr,
8045 serde_json::to_vec(&body).expect("HELLO body reserializes"),
8046 )
8047 .expect("HELLO frame builds")
8048 }
8049
8050 fn channel_request(channel: u16, corr: u64) -> Frame {
8051 Frame::build(
8052 FrameType::Request,
8053 Flags::new(true, Priority::Interactive, false),
8054 channel,
8055 0,
8056 corr,
8057 b"opaque".to_vec(),
8058 )
8059 .unwrap()
8060 }
8061
8062 fn route_ctx(
8063 connection_id: ConnectionId,
8064 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8065 let (tx, rx) = mpsc::channel(8);
8066 (
8067 RouteCtx {
8068 connection_id,
8069 egress: FrameSink::new(tx),
8070 },
8071 rx,
8072 )
8073 }
8074
8075 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8076 serde_json::from_slice(&frame.body).unwrap()
8077 }
8078
8079 async fn hello_via_sink(
8084 handler: &ControlHandler,
8085 ctx: &RouteCtx,
8086 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8087 hello: Frame,
8088 ) -> Frame {
8089 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8090 assert!(
8091 replies.is_empty(),
8092 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8093 );
8094 let ack = rx
8095 .try_recv()
8096 .expect("HELLO_ACK is queued on the module sink")
8097 .frame;
8098 assert_eq!(ack.header.ty, FrameType::HelloAck);
8099 ack
8100 }
8101
8102 fn parse_error(frame: &Frame) -> Value {
8103 serde_json::from_slice(&frame.body).unwrap()
8104 }
8105
8106 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8107 serde_json::from_slice(&frame.body).unwrap()
8108 }
8109
8110 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8111 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8112 route_channel,
8113 route_epoch: 0,
8114 kind,
8115 })
8116 .unwrap();
8117 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8118 }
8119
8120 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8121 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8122 module_id: module_id.to_string(),
8123 })
8124 .unwrap();
8125 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8126 }
8127
8128 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8129 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8130 }
8131
8132 fn route_open_frame_with_consumer_capabilities(
8133 corr: u64,
8134 module_id: &str,
8135 project_root: ScratchDir,
8136 consumer_capabilities: Option<Vec<String>>,
8137 ) -> Frame {
8138 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8139 target: RouteTarget::ToolProvider {
8140 module_id: module_id.to_string(),
8141 },
8142 identity: BindIdentity::new(
8143 project_root.path().to_path_buf(),
8144 "unit".to_string(),
8145 "session".to_string(),
8146 ),
8147 consumer_identity: None,
8148 consumer_capabilities,
8149 role_versions: None,
8150 admission_facts: None,
8151 scope: None,
8152 })
8153 .unwrap();
8154 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8155 }
8156
8157 fn route_open_frame_with_role_versions(
8158 corr: u64,
8159 module_id: &str,
8160 project_root: ScratchDir,
8161 role_versions: Option<BTreeMap<String, String>>,
8162 ) -> Frame {
8163 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8164 target: RouteTarget::ToolProvider {
8165 module_id: module_id.to_string(),
8166 },
8167 identity: BindIdentity::new(
8168 project_root.path().to_path_buf(),
8169 "unit".to_string(),
8170 format!("session-{corr}"),
8171 ),
8172 consumer_identity: None,
8173 consumer_capabilities: None,
8174 role_versions,
8175 admission_facts: None,
8176 scope: None,
8177 })
8178 .unwrap();
8179 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8180 }
8181
8182 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8183 entries
8184 .iter()
8185 .map(|(role, version)| (role.to_string(), version.to_string()))
8186 .collect()
8187 }
8188
8189 fn route_open_frame_with_admission_facts(
8190 corr: u64,
8191 module_id: &str,
8192 project_root: ScratchDir,
8193 consumer_identity: Option<subc_control::ConsumerIdentity>,
8194 facts: Option<Value>,
8195 ) -> Frame {
8196 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8197 target: RouteTarget::ToolProvider {
8198 module_id: module_id.to_string(),
8199 },
8200 identity: BindIdentity::new(
8201 project_root.path().to_path_buf(),
8202 "unit".to_string(),
8203 format!("session-{corr}"),
8204 ),
8205 consumer_identity,
8206 consumer_capabilities: None,
8207 role_versions: None,
8208 admission_facts: facts,
8209 scope: None,
8210 })
8211 .unwrap();
8212 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8213 }
8214
8215 #[derive(Clone, Default)]
8216 struct EventCapture {
8217 events: Arc<Mutex<Vec<CapturedEvent>>>,
8218 }
8219
8220 #[derive(Clone, Debug)]
8221 struct CapturedEvent {
8222 target: String,
8223 level: tracing::Level,
8224 fields: BTreeMap<String, String>,
8225 }
8226
8227 impl EventCapture {
8228 fn events(&self) -> Vec<CapturedEvent> {
8229 self.events.lock().unwrap().clone()
8230 }
8231 }
8232
8233 impl<S> Layer<S> for EventCapture
8234 where
8235 S: Subscriber,
8236 {
8237 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8238 let mut visitor = EventFieldVisitor::default();
8239 event.record(&mut visitor);
8240 self.events.lock().unwrap().push(CapturedEvent {
8241 target: event.metadata().target().to_string(),
8242 level: *event.metadata().level(),
8243 fields: visitor.fields,
8244 });
8245 }
8246 }
8247
8248 #[derive(Default)]
8249 struct EventFieldVisitor {
8250 fields: BTreeMap<String, String>,
8251 }
8252
8253 impl Visit for EventFieldVisitor {
8254 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8255 self.fields
8256 .insert(field.name().to_string(), format!("{value:?}"));
8257 }
8258 }
8259
8260 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8261 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8262 status,
8263 detail: Some("warming".to_string()),
8264 metrics: Some(json!({"queue_depth": 3})),
8265 })
8266 .unwrap();
8267 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8268 }
8269
8270 fn route_bind_ack(corr: u64) -> Frame {
8271 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8272 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8273 }
8274
8275 fn unique_project_root(label: &str) -> ScratchDir {
8276 ScratchDir::new(label)
8277 }
8278
8279 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8280 match parse_route_poll(frame) {
8281 ClientControlResponse::RoutePoll {
8282 status: None,
8283 live: Some(live),
8284 ..
8285 } => assert_eq!(live, expected_live),
8286 other => panic!("unexpected route.poll response: {other:?}"),
8287 }
8288 }
8289
8290 fn bind_liveness_route(
8291 registry: &Registry,
8292 forwarding: &ForwardingTable,
8293 module_id: &str,
8294 ) -> (RouteCtx, u16, u32) {
8295 let module_connection = ConnectionId::new(101);
8296 let client_connection = ConnectionId::new(202);
8297 let registration = registry
8298 .register_with_control_ops(
8299 manifest(module_id, PROTOCOL_VERSION),
8300 PROTOCOL_VERSION,
8301 module_connection,
8302 module_baseline_control_ops(),
8303 )
8304 .unwrap();
8305 let (module_tx, _module_rx) = mpsc::channel(8);
8306 let endpoint = forwarding
8307 .register_module_connection(
8308 module_connection,
8309 module_id.to_string(),
8310 PROTOCOL_VERSION,
8311 manifest_concurrency(®istration.manifest),
8312 FrameSink::new(module_tx),
8313 )
8314 .unwrap();
8315 let (client_ctx, _client_rx) = route_ctx(client_connection);
8316 let pending = forwarding
8317 .begin_route_bind_relay_for_test(
8318 client_connection,
8319 client_ctx.egress.clone(),
8320 1,
8321 module_id,
8322 )
8323 .unwrap();
8324 assert_eq!(pending.endpoint, endpoint);
8325 let route_channel = pending.client_channel;
8326 let route_epoch = pending.client_epoch;
8327 forwarding
8328 .complete_pending_relay(
8329 module_connection,
8330 pending.corr,
8331 RouteBindRelayOutcome::Accepted,
8332 )
8333 .unwrap();
8334 (client_ctx, route_channel, route_epoch)
8335 }
8336
8337 struct FakeProcessLiveness {
8338 live: Option<bool>,
8339 }
8340
8341 impl ModuleProcessLiveness for FakeProcessLiveness {
8342 fn process_live(&self, _module_id: &str) -> Option<bool> {
8343 self.live
8344 }
8345 }
8346
8347 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8348 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8349 {
8350 let registry = Arc::new(Registry::default());
8351 let supervisor_handle = SupervisorHandle::new();
8352 let supervisor = Supervisor::new_for_test(
8353 Arc::clone(®istry),
8354 RestartPolicy::new(1, Duration::from_millis(10)),
8355 )
8356 .with_handle(supervisor_handle.clone());
8357 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8358 let module = supervisor
8359 .spawn(ModuleSpec {
8360 module_id: "stderr-tail-wire".to_string(),
8361 program: fake_aft_stub_path(),
8362 args: Vec::new(),
8363 env: vec![
8364 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8365 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8366 ],
8367 reserved: false,
8368 reserved_prefixes: Vec::new(),
8369 protocol: ModuleProtocol::Subc,
8370 overlap: Default::default(),
8371 })
8372 .unwrap();
8373
8374 let deadline = Instant::now() + Duration::from_secs(5);
8375 loop {
8376 let tail = module.stderr_tail(None, None);
8377 if tail
8378 .entries
8379 .iter()
8380 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8381 && tail.entries.iter().any(|entry| {
8382 matches!(
8383 entry,
8384 TailEntry::Line {
8385 truncated: true,
8386 ..
8387 }
8388 )
8389 })
8390 {
8391 break;
8392 }
8393 assert!(
8394 Instant::now() < deadline,
8395 "module did not produce a truncated line and restart boundary: {tail:?}"
8396 );
8397 sleep(Duration::from_millis(10)).await;
8398 }
8399
8400 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8401 let request = ClientControlRequest::SupervisorStderrTail {
8402 module_id: "stderr-tail-wire".to_string(),
8403 max_lines: None,
8404 max_bytes: None,
8405 };
8406 let frame = Frame::build(
8407 FrameType::Request,
8408 control_flags(),
8409 0,
8410 0,
8411 1,
8412 serde_json::to_vec(&request).unwrap(),
8413 )
8414 .unwrap();
8415 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8416 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8417 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8418 serde_json::from_slice(&responses[0].body).unwrap()
8419 else {
8420 panic!("expected supervisor.stderr_tail response");
8421 };
8422
8423 assert!(
8424 tail.entries
8425 .iter()
8426 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8427 "the control response lost the restart boundary"
8428 );
8429 let Some(StderrTailEntry::Line {
8430 text,
8431 truncated,
8432 at_ms,
8433 }) = tail.entries.iter().find(|entry| {
8434 matches!(
8435 entry,
8436 StderrTailEntry::Line {
8437 truncated: true,
8438 ..
8439 }
8440 )
8441 })
8442 else {
8443 panic!("the control response lost the truncated line");
8444 };
8445 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8446 assert!(*truncated);
8447 assert!(
8448 at_ms.is_some(),
8449 "the control response lost the line's capture time"
8450 );
8451 }
8452
8453 #[tokio::test(flavor = "current_thread")]
8458 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8459 let dir = ScratchDir::new("terminals-off-worker");
8460 let journal_path = dir.join("terminals.jsonl");
8461 let registry = Arc::new(Registry::default());
8462 let supervisor_handle = SupervisorHandle::new();
8463 let supervisor =
8464 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8465 .with_handle(supervisor_handle.clone())
8466 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8467 let module = supervisor
8468 .spawn(ModuleSpec {
8469 module_id: "terminal-off-worker".to_string(),
8470 program: fake_aft_stub_path(),
8471 args: Vec::new(),
8472 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8473 reserved: false,
8474 reserved_prefixes: Vec::new(),
8475 protocol: ModuleProtocol::Subc,
8476 overlap: Default::default(),
8477 })
8478 .unwrap();
8479 let deadline = Instant::now() + Duration::from_secs(5);
8480 while module.terminal_history().entries.len() != 2 {
8481 assert!(Instant::now() < deadline, "module did not record two exits");
8482 sleep(Duration::from_millis(10)).await;
8483 }
8484
8485 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8486 let handler =
8487 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8488 let frame = Frame::build(
8489 FrameType::Request,
8490 control_flags(),
8491 0,
8492 0,
8493 1,
8494 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8495 module_id: "terminal-off-worker".to_string(),
8496 })
8497 .unwrap(),
8498 )
8499 .unwrap();
8500 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8501 let spawned_at = std::time::Instant::now();
8502 let read = tokio::spawn({
8503 let handler = Arc::clone(&handler);
8504 async move { handler.handle_control_frame(&ctx, frame).await }
8505 });
8506 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8509 .await
8510 .unwrap()
8511 .expect("the history read reached its pause");
8512 let elapsed = spawned_at.elapsed();
8513 assert!(
8514 elapsed < Duration::from_secs(2) && !read.is_finished(),
8515 "this task could not run while the history read was paused \
8516 (resumed after {elapsed:?}, read finished: {})",
8517 read.is_finished()
8518 );
8519
8520 drop(release);
8521 let responses = read.await.unwrap().unwrap();
8522 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8523 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8524 panic!("expected supervisor.terminals response");
8525 };
8526 assert_eq!(terminals.entries.len(), 2);
8527 assert_eq!(terminals.journal_skipped_lines, 0);
8528 assert_eq!(terminals.journal_read_errors, 0);
8529 }
8530
8531 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8532 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8533 let registry = Arc::new(Registry::default());
8534 let supervisor_handle = SupervisorHandle::new();
8535 let supervisor =
8536 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8537 .with_handle(supervisor_handle.clone());
8538 let module = supervisor
8539 .spawn(ModuleSpec {
8540 module_id: "terminal-golden".to_string(),
8541 program: fake_aft_stub_path(),
8542 args: Vec::new(),
8543 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8544 reserved: false,
8545 reserved_prefixes: Vec::new(),
8546 protocol: ModuleProtocol::Subc,
8547 overlap: Default::default(),
8548 })
8549 .unwrap();
8550
8551 let deadline = Instant::now() + Duration::from_secs(5);
8552 while module.terminal_history().entries.len() != 2 {
8553 assert!(
8554 Instant::now() < deadline,
8555 "module did not retain two terminal exits: {:?}",
8556 module.terminal_history()
8557 );
8558 sleep(Duration::from_millis(10)).await;
8559 }
8560
8561 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8562 let request = ClientControlRequest::SupervisorTerminals {
8563 module_id: "terminal-golden".to_string(),
8564 };
8565 let frame = Frame::build(
8566 FrameType::Request,
8567 control_flags(),
8568 0,
8569 0,
8570 1,
8571 serde_json::to_vec(&request).unwrap(),
8572 )
8573 .unwrap();
8574 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8575 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8576 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8577 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8578 panic!("expected supervisor.terminals response");
8579 };
8580 assert_eq!(terminals.entries.len(), 2);
8581 assert_eq!(terminals.dropped, 0);
8582
8583 let mut rendered = serde_json::to_value(response).unwrap();
8584 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8587 for (index, entry) in rendered["entries"]
8588 .as_array_mut()
8589 .expect("terminal response entries array")
8590 .iter_mut()
8591 .enumerate()
8592 {
8593 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8594 }
8595
8596 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8597 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8598 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8599 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8600 std::fs::write(&golden_path, &serialized).unwrap();
8601 }
8602 let expected: Value =
8603 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8604 assert_eq!(rendered, expected);
8605 }
8606
8607 #[test]
8608 fn hello_registers_manifest_and_returns_ack() {
8609 let registry = Arc::new(Registry::default());
8610 let handler = ControlHandler::new(Arc::clone(®istry));
8611 let conn = ConnectionId::new(1);
8612
8613 let responses = handler
8614 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8615 .unwrap();
8616
8617 assert_eq!(responses.len(), 1);
8618 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8619 assert_eq!(responses[0].header.channel, 0);
8620 assert_eq!(responses[0].header.corr, 7);
8621 let ack = parse_ack(&responses[0]);
8622 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8623 assert!(ack
8624 .subc_capabilities
8625 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8626 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8627 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8628 assert!(ack
8629 .subc_ops
8630 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8631 assert!(ack
8632 .subc_ops
8633 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8634
8635 let registration = registry.get_module("aft").unwrap().unwrap();
8636 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8637 assert_eq!(registration.state, ChannelState::Active);
8638 assert_eq!(registration.connection_id, conn);
8639 assert_eq!(registration.control_ops, module_baseline_control_ops());
8640 }
8641
8642 #[test]
8643 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8644 let invalid_identifiers = [
8645 ("case_change", "credentials-Provider/v1"),
8646 ("leading_zero", "credentials-provider/v01"),
8647 ("trailing_hyphen", "credentials-provider-/v1"),
8648 ("consecutive_hyphens", "credentials--provider/v1"),
8649 ("uppercase", "Credentials-provider/v1"),
8650 ("missing_v", "credentials-provider/1"),
8651 ("whitespace", "credentials provider/v1"),
8652 ("zero_version", "credentials-provider/v0"),
8653 ("out_of_range_version", "credentials-provider/v4294967296"),
8654 (
8655 "overlength_name",
8656 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8657 ),
8658 ];
8659 let mut cases = invalid_identifiers
8660 .into_iter()
8661 .map(|(name, identifier)| {
8662 (
8663 format!("identifier_{name}"),
8664 "capabilities.provides[0]".to_string(),
8665 identifier.to_string(),
8666 json!({ "provides": [identifier] }),
8667 None,
8668 )
8669 })
8670 .collect::<Vec<_>>();
8671 cases.extend([
8672 (
8673 "unknown_need".to_string(),
8674 "capabilities.requires[0].need".to_string(),
8675 "deferred".to_string(),
8676 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8677 None,
8678 ),
8679 (
8680 "duplicate_provides".to_string(),
8681 "capabilities.provides[1]".to_string(),
8682 "credentials-provider/v1".to_string(),
8683 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8684 None,
8685 ),
8686 (
8687 "duplicate_must_never_reach".to_string(),
8688 "capabilities.must_never_reach[1]".to_string(),
8689 "credentials-provider/v1".to_string(),
8690 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8691 None,
8692 ),
8693 (
8694 "duplicate_requires_same_need".to_string(),
8695 "capabilities.requires[1]".to_string(),
8696 "credentials-provider/v1".to_string(),
8697 json!({ "requires": [
8698 { "capability": "credentials-provider/v1", "need": "required" },
8699 { "capability": "credentials-provider/v1", "need": "required" }
8700 ] }),
8701 None,
8702 ),
8703 (
8704 "duplicate_requires_conflicting_need".to_string(),
8705 "capabilities.requires[1]".to_string(),
8706 "credentials-provider/v1".to_string(),
8707 json!({ "requires": [
8708 { "capability": "credentials-provider/v1", "need": "required" },
8709 { "capability": "credentials-provider/v1", "need": "optional" }
8710 ] }),
8711 None,
8712 ),
8713 (
8714 "capabilities_root_pointer".to_string(),
8715 "runtime_computed[0]".to_string(),
8716 "/capabilities".to_string(),
8717 json!({}),
8718 Some(json!(["/capabilities"])),
8719 ),
8720 (
8721 "capabilities_descendant_pointer".to_string(),
8722 "runtime_computed[0]".to_string(),
8723 "/capabilities/provides".to_string(),
8724 json!({}),
8725 Some(json!(["/capabilities/provides"])),
8726 ),
8727 (
8728 "malformed_pointer_without_leading_slash".to_string(),
8729 "runtime_computed[0]".to_string(),
8730 "capabilities".to_string(),
8731 json!({}),
8732 Some(json!(["capabilities"])),
8733 ),
8734 (
8735 "malformed_pointer_escape".to_string(),
8736 "runtime_computed[0]".to_string(),
8737 "/roles/~2/tools".to_string(),
8738 json!({}),
8739 Some(json!(["/roles/~2/tools"])),
8740 ),
8741 (
8742 "unknown_capabilities_field".to_string(),
8743 "capabilities.future".to_string(),
8744 "<array>".to_string(),
8745 json!({ "future": [] }),
8746 None,
8747 ),
8748 ]);
8749
8750 for (index, (name, field, value, capabilities, runtime_computed)) in
8751 cases.into_iter().enumerate()
8752 {
8753 let registry = Arc::new(Registry::default());
8754 let handler = ControlHandler::new(Arc::clone(®istry));
8755 let response = handler
8756 .handle_control(
8757 ConnectionId::new((index + 1) as u64),
8758 capability_grammar_hello_frame(
8759 capabilities,
8760 runtime_computed,
8761 index as u64 + 1,
8762 ),
8763 )
8764 .expect("invalid HELLO returns a refusal");
8765
8766 assert_eq!(response.len(), 1, "{name} must emit one refusal");
8767 let error = parse_error(&response[0]);
8768 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8769 let message = error["message"]
8770 .as_str()
8771 .expect("error message is a string");
8772 assert!(
8773 message.contains(&field),
8774 "{name}: field missing from {message}"
8775 );
8776 assert!(
8777 message.contains(&value),
8778 "{name}: value missing from {message}"
8779 );
8780 assert_eq!(
8781 registry
8782 .active_registration_count()
8783 .expect("registry reads"),
8784 0,
8785 "{name}: refused HELLO must not create a catalog entry"
8786 );
8787 }
8788 }
8789
8790 #[test]
8791 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8792 let registry = Arc::new(Registry::default());
8793 let handler = ControlHandler::new(Arc::clone(®istry));
8794 let capabilities = json!({
8795 "provides": ["credentials-provider/v1"],
8796 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8797 "must_never_reach": ["federation-transport/v1"]
8798 });
8799 let response = handler
8800 .handle_control(
8801 ConnectionId::new(99),
8802 capability_grammar_hello_frame(
8803 capabilities.clone(),
8804 Some(json!(["/roles/0/tools"])),
8805 99,
8806 ),
8807 )
8808 .expect("valid HELLO registers");
8809 assert_eq!(response[0].header.ty, FrameType::HelloAck);
8810
8811 let request = Frame::build(
8812 FrameType::Request,
8813 control_flags(),
8814 0,
8815 0,
8816 100,
8817 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8818 .expect("catalog request serializes"),
8819 )
8820 .expect("catalog request frame builds");
8821 let response = handler
8822 .handle_catalog_list(request, None)
8823 .expect("catalog list succeeds");
8824 let ClientControlResponse::CatalogList { modules, .. } =
8825 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8826 else {
8827 panic!("catalog request must return catalog.list");
8828 };
8829 assert_eq!(modules.len(), 1);
8830 assert_eq!(
8831 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8832 capabilities
8833 );
8834 }
8835
8836 #[test]
8837 fn catalog_list_mirrors_management_operation_description() {
8838 let registry = Arc::new(Registry::default());
8839 let handler = ControlHandler::new(Arc::clone(®istry));
8840 let description = "List managed records and return their identifiers and metadata.";
8841 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8842 manifest.provides = vec![ProviderRole::ManagementSurface {
8843 operations: vec![ManagementOperation {
8844 name: "records.list".to_string(),
8845 kind: ManagementOperationKind::Query,
8846 description: Some(description.to_string()),
8847 }],
8848 config_schema: json!({"type": "object"}),
8849 observability: vec![ObservabilitySurface {
8850 name: "records.stats".to_string(),
8851 kind: ObservabilityKind::Snapshot,
8852 }],
8853 identity_scope: vec![IdentityScope::Project],
8854 concurrency: Concurrency::ModuleManaged,
8855 }];
8856 registry
8857 .register_with_control_ops(
8858 manifest,
8859 PROTOCOL_VERSION,
8860 ConnectionId::new(99),
8861 Vec::new(),
8862 )
8863 .expect("described management manifest registers");
8864
8865 let request = Frame::build(
8866 FrameType::Request,
8867 control_flags(),
8868 0,
8869 0,
8870 100,
8871 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8872 .expect("catalog request serializes"),
8873 )
8874 .expect("catalog request frame builds");
8875 let response = handler
8876 .handle_catalog_list(request, None)
8877 .expect("catalog list succeeds");
8878 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8879 assert_eq!(
8880 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8881 "catalog.list must preserve the declared operation description verbatim"
8882 );
8883 }
8884
8885 #[test]
8886 fn catalog_list_mirrors_registered_operation_delivery() {
8887 let registry = Arc::new(Registry::default());
8888 let handler = ControlHandler::new(Arc::clone(®istry));
8889 let delivery =
8890 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
8891 let mut body = serde_json::to_value(ModuleHelloBody {
8892 manifest: manifest("watch-management", PROTOCOL_VERSION),
8893 protocol_ver: PROTOCOL_VERSION,
8894 control_ops: None,
8895 launch_nonce: None,
8896 })
8897 .unwrap();
8898 body["manifest"]["provides"] = json!([{
8899 "role": "management_surface",
8900 "operations": [
8901 {"name": "status", "kind": "query"},
8902 {"name": "future", "kind": "query"},
8903 {"name": "reset", "kind": "mutate"}
8904 ],
8905 "config_schema": {}, "observability": [], "identity_scope": []
8906 }]);
8907 body["manifest"]["operation_delivery"] = delivery.clone();
8908 let hello = Frame::build(
8909 FrameType::Hello,
8910 control_flags(),
8911 0,
8912 0,
8913 99,
8914 serde_json::to_vec(&body).unwrap(),
8915 )
8916 .unwrap();
8917 let response = handler
8918 .handle_control(ConnectionId::new(99), hello)
8919 .unwrap();
8920 assert_eq!(response[0].header.ty, FrameType::HelloAck);
8921
8922 let (_, registrations) = registry.list_modules().unwrap();
8923 let registered = ®istrations[0].manifest;
8924 assert!(registered.is_watch_operation("status"));
8925 assert!(!registered.is_watch_operation("future"));
8926 assert!(!registered.is_watch_operation("reset"));
8927 assert!(!registered.is_watch_operation("missing"));
8928
8929 let request = Frame::build(
8930 FrameType::Request,
8931 control_flags(),
8932 0,
8933 0,
8934 100,
8935 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
8936 )
8937 .unwrap();
8938 let response = handler.handle_catalog_list(request, None).unwrap();
8939 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
8940 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
8941 let ClientControlResponse::CatalogList { modules, .. } =
8942 serde_json::from_slice(&response[0].body).unwrap()
8943 else {
8944 panic!("expected catalog.list");
8945 };
8946 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
8947 }
8948
8949 #[test]
8950 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8951 let registry = Arc::new(Registry::default());
8952 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8953 [("vault".to_string(), true), ("squatter".to_string(), true)],
8954 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8955 );
8956 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8957 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8958 provides: vec!["credentials-provider/v1".to_string()],
8959 requires: Vec::new(),
8960 must_never_reach: Vec::new(),
8961 });
8962 let frame = Frame::build(
8963 FrameType::Hello,
8964 control_flags(),
8965 0,
8966 0,
8967 77,
8968 serde_json::to_vec(&ModuleHelloBody {
8969 manifest: squatter,
8970 protocol_ver: PROTOCOL_VERSION,
8971 control_ops: None,
8972 launch_nonce: None,
8973 })
8974 .expect("HELLO serializes"),
8975 )
8976 .expect("HELLO frame builds");
8977 let response = handler
8978 .handle_control(ConnectionId::new(77), frame)
8979 .expect("reserved claim receives a typed refusal");
8980 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8981 assert_eq!(
8982 registry
8983 .active_registration_count()
8984 .expect("registry reads"),
8985 0,
8986 "a reserved capability refusal must not leave a catalog entry"
8987 );
8988 }
8989
8990 #[test]
8991 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8992 for settlement in ["timeout", "inconclusive", "drop"] {
8993 let breakers = RouteBindBreakers::default();
8994 let RouteBindAdmission::Admitted {
8995 guard: mut old,
8996 probe: false,
8997 } = breakers.admit("prov")
8998 else {
8999 panic!("ordinary relay admitted")
9000 };
9001 let RouteBindAdmission::Admitted {
9002 guard: mut opener, ..
9003 } = breakers.admit("prov")
9004 else {
9005 panic!("second relay admitted")
9006 };
9007 assert!(
9008 !opener
9009 .record_timeout(1, Duration::ZERO)
9010 .unwrap()
9011 .reopened_after_probe
9012 );
9013 let RouteBindAdmission::Admitted {
9014 guard: mut probe,
9015 probe: true,
9016 } = breakers.admit("prov")
9017 else {
9018 panic!("one cooldown probe admitted")
9019 };
9020 match settlement {
9021 "timeout" => assert!(
9022 !old.record_timeout(1, Duration::ZERO)
9023 .unwrap()
9024 .reopened_after_probe
9025 ),
9026 "inconclusive" => old.record_inconclusive(),
9027 "drop" => drop(old),
9028 _ => unreachable!(),
9029 }
9030 assert!(
9031 matches!(
9032 breakers.admit("prov"),
9033 RouteBindAdmission::Refused {
9034 probe_in_flight: true,
9035 ..
9036 }
9037 ),
9038 "{settlement} of a pre-open relay cannot release the real probe"
9039 );
9040 assert!(
9041 probe
9042 .record_timeout(1, Duration::ZERO)
9043 .unwrap()
9044 .reopened_after_probe
9045 );
9046 assert!(matches!(
9047 breakers.admit("prov"),
9048 RouteBindAdmission::Admitted { probe: true, .. }
9049 ));
9050 }
9051 let breakers = RouteBindBreakers::default();
9052 let admit = || match breakers.admit("prov") {
9053 RouteBindAdmission::Admitted { guard, .. } => guard,
9054 _ => panic!("relay admitted"),
9055 };
9056 admit().record_timeout(1, Duration::ZERO);
9057 let mut old_probe = admit();
9058 breakers.reset_for_new_module_connection("prov");
9059 admit().record_timeout(1, Duration::ZERO);
9060 let _new_probe = admit();
9061 old_probe.record_inconclusive();
9062 assert!(matches!(
9063 breakers.admit("prov"),
9064 RouteBindAdmission::Refused {
9065 probe_in_flight: true,
9066 ..
9067 }
9068 ));
9069 }
9070
9071 #[tokio::test]
9072 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9073 for candidate in [false, true] {
9074 let registry = Arc::new(Registry::default());
9075 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9076 [("vault".to_string(), true), ("squatter".to_string(), true)],
9077 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9078 );
9079 let conn = ConnectionId::new(77);
9080 let (ctx, mut rx) = route_ctx(conn);
9081 let initial = capability_manifest("squatter", &[], &[]);
9082 if candidate {
9083 registry
9084 .register_candidate_with_control_ops(
9085 initial.clone(),
9086 PROTOCOL_VERSION,
9087 conn,
9088 module_baseline_control_ops(),
9089 )
9090 .unwrap();
9091 handler
9092 .forwarding
9093 .register_candidate_module_connection(
9094 conn,
9095 "squatter".to_string(),
9096 PROTOCOL_VERSION,
9097 manifest_concurrency(&initial),
9098 ctx.egress.clone(),
9099 )
9100 .unwrap();
9101 } else {
9102 hello_via_sink(
9103 &handler,
9104 &ctx,
9105 &mut rx,
9106 hello_frame_with_manifest(initial.clone(), 1),
9107 )
9108 .await;
9109 }
9110 let response = handler
9111 .handle_control_frame(
9112 &ctx,
9113 catalog_update_with_capabilities_frame(
9114 2,
9115 capability_manifest("squatter", &["credentials-provider/v1"], &[])
9116 .capabilities
9117 .unwrap(),
9118 ),
9119 )
9120 .await
9121 .unwrap();
9122 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9123 assert_eq!(
9124 registry
9125 .get_module_by_connection(conn)
9126 .unwrap()
9127 .unwrap()
9128 .manifest,
9129 initial
9130 );
9131 }
9132 }
9133
9134 #[test]
9135 fn server_describe_surfaces_required_capability_verdict_fields() {
9136 let registry = Arc::new(Registry::default());
9137 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9138 [
9139 ("consumer".to_string(), true),
9140 ("provider".to_string(), false),
9141 ],
9142 BTreeMap::new(),
9143 );
9144 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9145 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9146 provides: Vec::new(),
9147 requires: vec![subc_protocol::manifest::CapabilityRequirement {
9148 capability: "credentials-provider/v1".to_string(),
9149 need: subc_protocol::manifest::CapabilityNeed::Required,
9150 }],
9151 must_never_reach: Vec::new(),
9152 });
9153 let hello = Frame::build(
9154 FrameType::Hello,
9155 control_flags(),
9156 0,
9157 0,
9158 78,
9159 serde_json::to_vec(&ModuleHelloBody {
9160 manifest: consumer,
9161 protocol_ver: PROTOCOL_VERSION,
9162 control_ops: None,
9163 launch_nonce: None,
9164 })
9165 .expect("HELLO serializes"),
9166 )
9167 .expect("HELLO frame builds");
9168 handler
9169 .handle_control(ConnectionId::new(78), hello)
9170 .expect("consumer registers");
9171 let describe = Frame::build(
9172 FrameType::Request,
9173 control_flags(),
9174 0,
9175 0,
9176 79,
9177 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
9178 .expect("request serializes"),
9179 )
9180 .expect("describe frame builds");
9181 let response = handler
9182 .handle_server_describe(describe)
9183 .expect("server.describe succeeds");
9184 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
9185 let requirement = &rendered["capability_requirements"][0];
9186 assert_eq!(requirement["consumer"], "consumer");
9187 assert_eq!(requirement["verdict"], "never_provided");
9188 assert_eq!(requirement["episode_seq"], 1);
9189 assert_eq!(requirement["config_satisfiable"], false);
9190 assert_eq!(requirement["runtime_available"], false);
9191 assert!(requirement["detail"]
9192 .as_str()
9193 .expect("detail string")
9194 .contains("credentials-provider/v1"));
9195 }
9196
9197 fn deregister_required_provider(
9203 daemon_shutdown_begun: bool,
9204 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
9205 const CAPABILITY: &str = "project-identity/v1";
9206 let registry = Arc::new(Registry::default());
9207 let daemon_shutdown = DaemonShutdownFlag::default();
9208 let handler = ControlHandler::new(Arc::clone(®istry))
9209 .with_capability_config(
9210 [
9211 ("consumer".to_string(), true),
9212 ("provider".to_string(), true),
9213 ],
9214 BTreeMap::new(),
9215 )
9216 .with_daemon_shutdown_flag(daemon_shutdown.clone());
9217 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9218 consumer.capabilities = Some(CapabilityDeclarations {
9219 provides: Vec::new(),
9220 requires: vec![subc_protocol::manifest::CapabilityRequirement {
9221 capability: CAPABILITY.to_string(),
9222 need: subc_protocol::manifest::CapabilityNeed::Required,
9223 }],
9224 must_never_reach: Vec::new(),
9225 });
9226 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
9227 let provider_conn = ConnectionId::new(182);
9228 for (conn, manifest, corr) in [
9229 (ConnectionId::new(181), consumer, 181),
9230 (provider_conn, provider, 182),
9231 ] {
9232 let hello = handler
9233 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
9234 .expect("module registers");
9235 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9236 }
9237 assert_eq!(
9238 handler.capability_evaluator.verdict("consumer", CAPABILITY),
9239 Some(CapabilityVerdict::Provided),
9240 "the registered provider must satisfy the requirement first"
9241 );
9242 if daemon_shutdown_begun {
9243 daemon_shutdown.set_for_test();
9244 }
9245
9246 let capture = EventCapture::default();
9247 let _subscriber =
9248 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9249 handler
9250 .cleanup_connection(provider_conn)
9251 .expect("provider deregisters");
9252 let events = capture
9253 .events()
9254 .into_iter()
9255 .filter(|event| {
9256 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
9257 })
9258 .collect();
9259 (
9260 events,
9261 handler.capability_evaluator.verdict("consumer", CAPABILITY),
9262 )
9263 }
9264
9265 #[test]
9266 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
9267 let (events, verdict) = deregister_required_provider(true);
9268 assert_eq!(
9269 verdict,
9270 Some(CapabilityVerdict::NeverProvided),
9271 "shutdown must not change the verdict route.open refusals read"
9272 );
9273 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9274 let event = &events[0];
9275 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
9276 assert_eq!(
9277 event.fields.get("cause").map(String::as_str),
9278 Some("\"daemon_shutdown\""),
9279 "{event:?}"
9280 );
9281 assert_eq!(
9282 event.fields.get("verdict").map(String::as_str),
9283 Some("\"never_provided\""),
9284 "{event:?}"
9285 );
9286 }
9287
9288 #[test]
9289 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
9290 let (events, verdict) = deregister_required_provider(false);
9291 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
9292 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
9293 let event = &events[0];
9294 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
9295 assert_eq!(event.fields.get("cause"), None, "{event:?}");
9296 assert_eq!(
9297 event.fields.get("verdict").map(String::as_str),
9298 Some("\"never_provided\""),
9299 "{event:?}"
9300 );
9301 }
9302
9303 #[test]
9304 fn catalog_list_omits_capabilities_for_legacy_manifest() {
9305 let registry = Arc::new(Registry::default());
9306 let handler = ControlHandler::new(Arc::clone(®istry));
9307 let hello = handler
9308 .handle_control(
9309 ConnectionId::new(101),
9310 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
9311 )
9312 .expect("legacy HELLO registers");
9313 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9314
9315 let request = Frame::build(
9316 FrameType::Request,
9317 control_flags(),
9318 0,
9319 0,
9320 102,
9321 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9322 .expect("catalog request serializes"),
9323 )
9324 .expect("catalog request frame builds");
9325 let response = handler
9326 .handle_catalog_list(request, None)
9327 .expect("catalog list succeeds");
9328 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9329 assert!(
9330 body["modules"][0].get("capabilities").is_none(),
9331 "legacy manifest must retain an absent capabilities field on catalog.list"
9332 );
9333 }
9334
9335 #[test]
9336 fn hello_ack_omits_storage_when_no_storage_config() {
9337 let registry = Arc::new(Registry::default());
9338 let handler = ControlHandler::new(Arc::clone(®istry));
9339 let responses = handler
9340 .handle_control(
9341 ConnectionId::new(1),
9342 hello_frame("aft", PROTOCOL_VERSION, 7),
9343 )
9344 .unwrap();
9345 let ack = parse_ack(&responses[0]);
9346 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
9347 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
9348 }
9349
9350 #[tokio::test]
9351 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
9352 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
9353 let registry = Arc::new(Registry::default());
9354 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
9355 let responses = handler
9356 .handle_control(
9357 ConnectionId::new(1),
9358 hello_frame("aft", PROTOCOL_VERSION, 7),
9359 )
9360 .unwrap();
9361 let ack = parse_ack(&responses[0]);
9362 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
9363
9364 let described = handler
9365 .handle_control_frame(
9366 &route_ctx(ConnectionId::new(2)).0,
9367 Frame::build(
9368 FrameType::Request,
9369 control_flags(),
9370 0,
9371 0,
9372 9,
9373 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9374 )
9375 .unwrap(),
9376 )
9377 .await
9378 .unwrap();
9379 let ClientControlResponse::ServerDescribe { machine_id, .. } =
9380 serde_json::from_slice(&described[0].body).unwrap()
9381 else {
9382 panic!("server.describe answered with another shape");
9383 };
9384 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
9385 }
9386
9387 #[test]
9388 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
9389 let registry = Arc::new(Registry::default());
9392 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
9393 crate::daemon_config::StorageConfig::Sqlite {
9394 data_home: std::path::PathBuf::from("/data"),
9395 },
9396 ));
9397
9398 let responses = handler
9399 .handle_control(
9400 ConnectionId::new(1),
9401 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
9402 )
9403 .unwrap();
9404 let ack = parse_ack(&responses[0]);
9405 assert_eq!(
9406 ack.storage,
9407 Some(serde_json::json!({
9408 "module_id": "alfonso-routing",
9409 "storage_namespace": "default",
9410 "isolation": { "kind": "module" },
9411 "backend": {
9412 "backend": "sqlite",
9413 "path": "/data/cortexkit/alfonso-routing/store.db"
9414 }
9415 })),
9416 "the delivered descriptor is the module's own sqlite store path"
9417 );
9418 }
9419
9420 #[test]
9421 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
9422 let registry = Arc::new(Registry::default());
9423 let handler = ControlHandler::new(Arc::clone(®istry));
9424 let conn = ConnectionId::new(1);
9425 let responses = handler
9426 .handle_control(
9427 conn,
9428 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9429 )
9430 .unwrap();
9431 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9432 let registration = registry.get_module("aft").unwrap().unwrap();
9433 assert_eq!(registration.control_ops, module_baseline_control_ops());
9434
9435 let frame =
9436 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
9437 assert!(handler
9438 .guard_module_control_op(&frame, "aft", "route.bind")
9439 .unwrap()
9440 .is_none());
9441 let error = handler
9442 .guard_module_control_op(&frame, "aft", "test.synthetic")
9443 .unwrap()
9444 .expect("synthetic ungranted op should be rejected");
9445 assert_eq!(error.header.ty, FrameType::Error);
9446 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
9447 }
9448
9449 #[test]
9450 fn hello_control_ops_some_adds_optional_grants() {
9451 let registry = Arc::new(Registry::default());
9452 let handler = ControlHandler::new(Arc::clone(®istry));
9453 handler
9454 .handle_control(
9455 ConnectionId::new(1),
9456 hello_frame_with_control_ops(
9457 "aft",
9458 PROTOCOL_VERSION,
9459 7,
9460 Some(vec![
9461 "future.synthetic".to_string(),
9462 "route.bind".to_string(),
9463 ]),
9464 ),
9465 )
9466 .unwrap();
9467 let registration = registry.get_module("aft").unwrap().unwrap();
9468 assert_eq!(
9469 registration.control_ops,
9470 vec![
9471 "route.bind".to_string(),
9472 "route.status".to_string(),
9473 "future.synthetic".to_string(),
9474 ]
9475 );
9476 let frame =
9477 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9478 assert!(handler
9479 .guard_module_control_op(&frame, "aft", "future.synthetic")
9480 .unwrap()
9481 .is_none());
9482 }
9483
9484 #[tokio::test]
9485 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9486 let registry = Arc::new(Registry::default());
9487 let forwarding = Arc::new(ForwardingTable::default());
9488 let handler =
9489 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9490 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9491 hello_via_sink(
9492 &handler,
9493 &module_ctx,
9494 &mut module_rx,
9495 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9496 )
9497 .await;
9498
9499 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9500 let responses = handler
9501 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9502 .await
9503 .unwrap();
9504 assert_eq!(responses.len(), 1);
9505 assert_eq!(responses[0].header.ty, FrameType::Error);
9506 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9507 assert!(module_rx.try_recv().is_err());
9508 }
9509
9510 #[tokio::test]
9511 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9512 let registry = Arc::new(Registry::default());
9513 let forwarding = Arc::new(ForwardingTable::default());
9514 let handler =
9515 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9516 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9517 hello_via_sink(
9518 &handler,
9519 &module_ctx,
9520 &mut module_rx,
9521 hello_frame_with_control_ops(
9522 "aft",
9523 PROTOCOL_VERSION,
9524 7,
9525 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9526 ),
9527 )
9528 .await;
9529
9530 let project_root = unique_project_root("demux");
9531 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9532 let route_handler = handler.clone();
9533 let route_task = tokio::spawn(async move {
9534 route_handler
9535 .handle_control_frame(
9536 &route_client_ctx,
9537 route_open_frame(100, "aft", project_root),
9538 )
9539 .await
9540 .unwrap()
9541 });
9542 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9543 .await
9544 .unwrap()
9545 .unwrap();
9546 assert!(matches!(
9547 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
9548 ModuleControlRequest::RouteBind { .. }
9549 ));
9550
9551 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
9552 let health_handler = handler.clone();
9553 let health_task = tokio::spawn(async move {
9554 health_handler
9555 .handle_control_frame(
9556 &health_client_ctx,
9557 supervisor_health_probe_frame(101, "aft"),
9558 )
9559 .await
9560 .unwrap()
9561 });
9562 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9563 .await
9564 .unwrap()
9565 .unwrap();
9566 assert_eq!(
9567 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9568 ModuleControlRequest::HealthCheck {}
9569 );
9570
9571 handler
9572 .handle_control_frame(
9573 &module_ctx,
9574 health_response(health_frame.header.corr, HealthStatus::Degraded),
9575 )
9576 .await
9577 .unwrap();
9578 let health_response = health_task.await.unwrap();
9579 assert_eq!(health_response.len(), 1);
9580 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9581 ClientControlResponse::SupervisorHealthProbe {
9582 module_id,
9583 status,
9584 detail,
9585 metrics,
9586 } => {
9587 assert_eq!(module_id, "aft");
9588 assert_eq!(status, HealthStatus::Degraded);
9589 assert_eq!(detail.as_deref(), Some("warming"));
9590 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
9591 }
9592 other => panic!("unexpected health response: {other:?}"),
9593 }
9594
9595 handler
9596 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9597 .await
9598 .unwrap();
9599 let route_response = route_task.await.unwrap();
9600 assert!(route_response.is_empty());
9601 let published = route_client_rx.recv().await.unwrap();
9602 assert!(matches!(
9603 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9604 ClientControlResponse::RouteOpen { .. }
9605 ));
9606 }
9607
9608 async fn relay_route_open(
9613 handler: &ControlHandler,
9614 client_connection: ConnectionId,
9615 client_egress: &FrameSink,
9616 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9617 corr: u64,
9618 module_id: &str,
9619 project_root_label: &str,
9620 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
9621 let ctx = RouteCtx {
9622 connection_id: client_connection,
9623 egress: client_egress.clone(),
9624 };
9625 let handler = handler.clone();
9626 let project_root = unique_project_root(project_root_label);
9627 let module_id = module_id.to_string();
9628 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
9629 let task = tokio::spawn(async move {
9630 let _guard = tracing::dispatcher::set_default(&dispatch);
9631 handler
9632 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
9633 .await
9634 .unwrap()
9635 });
9636 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
9637 .await
9638 .expect("module receives the relayed route.bind")
9639 .expect("module egress is open");
9640 (task, bind.frame)
9641 }
9642
9643 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
9644 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
9645 ModuleControlRequest::RouteBind {
9646 route_channel,
9647 epoch,
9648 ..
9649 } => (route_channel, epoch),
9650 other => panic!("expected a route.bind request, got {other:?}"),
9651 }
9652 }
9653
9654 fn published_route(frame: &Frame) -> (u16, u32) {
9655 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
9656 ClientControlResponse::RouteOpen {
9657 route_channel,
9658 route_epoch,
9659 } => (route_channel, route_epoch),
9660 other => panic!("expected a route.open response, got {other:?}"),
9661 }
9662 }
9663
9664 #[tokio::test]
9684 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
9685 let registry = Arc::new(Registry::default());
9686 let forwarding = Arc::new(ForwardingTable::default());
9687 let handler =
9688 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9689
9690 let module_connection = ConnectionId::new(30);
9691 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9692 hello_via_sink(
9693 &handler,
9694 &module_ctx,
9695 &mut module_rx,
9696 hello_frame("aft", PROTOCOL_VERSION, 7),
9697 )
9698 .await;
9699
9700 let dying_client = ConnectionId::new(31);
9701 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
9702
9703 let (first_task, first_bind) = relay_route_open(
9706 &handler,
9707 dying_client,
9708 &dying_ctx.egress,
9709 &mut module_rx,
9710 100,
9711 "aft",
9712 "closing-first",
9713 )
9714 .await;
9715 handler
9716 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
9717 .await
9718 .unwrap();
9719 assert!(first_task.await.unwrap().is_empty());
9720 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
9721
9722 let (second_task, second_bind) = relay_route_open(
9724 &handler,
9725 dying_client,
9726 &dying_ctx.egress,
9727 &mut module_rx,
9728 101,
9729 "aft",
9730 "closing-second",
9731 )
9732 .await;
9733 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9734
9735 assert!(forwarding
9738 .escalate_client_delivery_failure(
9739 dying_client,
9740 first_channel,
9741 first_epoch,
9742 CloseReason::new(
9743 "module_to_client_delivery_failed",
9744 "client egress refused a module frame",
9745 ),
9746 crate::forwarding::UndeliveredFrame {
9747 module_id: None,
9748 sink: &dying_ctx.egress,
9749 },
9750 )
9751 .unwrap());
9752 assert!(!dying_ctx.egress.is_closed());
9753
9754 let ack = handler
9756 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9757 .await;
9758 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9759 if module_loop_error.is_some() {
9760 handler.cleanup_connection(module_connection).unwrap();
9764 }
9765 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9769 .await
9770 .ok()
9771 .flatten();
9772
9773 assert!(
9775 registry
9776 .get_module_by_connection(module_connection)
9777 .unwrap()
9778 .is_some(),
9779 "one client's closing connection ended the shared module connection: \
9780 {module_loop_error:?}"
9781 );
9782 let cotenant = ConnectionId::new(32);
9784 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9785 let (cotenant_task, cotenant_bind) = relay_route_open(
9786 &handler,
9787 cotenant,
9788 &cotenant_ctx.egress,
9789 &mut module_rx,
9790 102,
9791 "aft",
9792 "closing-cotenant",
9793 )
9794 .await;
9795 handler
9796 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9797 .await
9798 .unwrap();
9799 assert!(cotenant_task.await.unwrap().is_empty());
9800 let (cotenant_channel, cotenant_epoch) =
9801 published_route(&cotenant_rx.recv().await.unwrap());
9802 assert!(matches!(
9803 forwarding
9804 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9805 .unwrap(),
9806 DataRoute::Client(DataRouteState::Bound(_))
9807 ));
9808
9809 let goodbye = post_ack_module_frame
9812 .expect("module receives a GOODBYE for the abandoned route channel");
9813 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9814 assert_eq!(goodbye.header.channel, abandoned_channel);
9815 assert_eq!(goodbye.header.epoch, abandoned_epoch);
9816
9817 assert!(dying_rx.try_recv().is_err());
9821 let second_response = second_task.await.unwrap();
9822 assert_eq!(second_response.len(), 1);
9823 assert_eq!(
9824 parse_error(&second_response[0])["code"],
9825 "target_unavailable"
9826 );
9827 }
9828
9829 async fn catalog_snapshot(
9832 handler: &ControlHandler,
9833 client_ctx: &RouteCtx,
9834 corr: u64,
9835 ) -> (u64, Vec<(String, bool)>) {
9836 let body =
9837 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
9838 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
9839 let reply = handler
9840 .handle_control_frame(client_ctx, request)
9841 .await
9842 .unwrap()
9843 .pop()
9844 .unwrap();
9845 let ClientControlResponse::CatalogList {
9846 generation,
9847 modules,
9848 ..
9849 } = serde_json::from_slice(&reply.body).unwrap()
9850 else {
9851 panic!("not a catalog.list reply");
9852 };
9853 (
9854 generation,
9855 modules
9856 .into_iter()
9857 .map(|entry| (entry.module_id, entry.ready))
9858 .collect(),
9859 )
9860 }
9861
9862 fn hello_held_before_forwarding() -> (
9868 ControlHandler,
9869 Arc<std::sync::Barrier>,
9870 std::thread::JoinHandle<Vec<Frame>>,
9871 RouteCtx,
9872 mpsc::Receiver<crate::router::OutboundFrame>,
9873 ) {
9874 let hold = Arc::new(std::sync::Barrier::new(2));
9875 let handler = ControlHandler::with_forwarding(
9876 Arc::new(Registry::default()),
9877 Arc::new(ForwardingTable::default()),
9878 )
9879 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
9880 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9881 let hello = {
9882 let handler = handler.clone();
9883 let module_ctx = module_ctx.clone();
9884 std::thread::spawn(move || {
9885 handler
9886 .handle_hello(
9887 module_ctx.connection_id,
9888 Some(module_ctx.egress.clone()),
9889 hello_frame("aft", PROTOCOL_VERSION, 7),
9890 )
9891 .unwrap()
9892 })
9893 };
9894 (handler, hold, hello, module_ctx, module_rx)
9895 }
9896
9897 #[tokio::test]
9906 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
9907 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
9908 let client = ConnectionId::new(41);
9909 let (client_ctx, mut client_rx) = route_ctx(client);
9910
9911 hold.wait();
9912 assert!(handler.registry.get_module("aft").unwrap().is_some());
9914 assert!(!handler
9915 .forwarding
9916 .has_live_module_connection("aft")
9917 .unwrap());
9918 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
9919 assert!(
9920 listed.is_empty(),
9921 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
9922 );
9923 let refused = handler
9924 .handle_control_frame(
9925 &client_ctx,
9926 route_open_frame(301, "aft", unique_project_root("listed-window")),
9927 )
9928 .await
9929 .unwrap();
9930 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
9931 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
9932 let duplicate = handler
9933 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
9934 .await
9935 .unwrap();
9936 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9937
9938 hold.wait();
9939 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9940 let ack = module_rx.try_recv().unwrap().frame;
9941 assert_eq!(ack.header.ty, FrameType::HelloAck);
9942 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9943 assert_eq!(listed, vec![("aft".to_string(), true)]);
9944
9945 let (open_task, bind) = relay_route_open(
9946 &handler,
9947 client,
9948 &client_ctx.egress,
9949 &mut module_rx,
9950 303,
9951 "aft",
9952 "listed-after",
9953 )
9954 .await;
9955 assert!(handler
9956 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9957 .await
9958 .unwrap()
9959 .is_empty());
9960 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
9961 published_route(&client_rx.try_recv().unwrap().frame);
9962 }
9963
9964 #[tokio::test]
9971 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
9972 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
9973 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9974 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9975
9976 hold.wait();
9977 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
9978 assert!(listed.is_empty());
9979 assert_eq!(
9980 during, before,
9981 "the generation moved while the module was not listed"
9982 );
9983
9984 hold.wait();
9985 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9986 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9987 assert_eq!(listed, vec![("aft".to_string(), true)]);
9988 assert_eq!(
9989 after,
9990 before + 1,
9991 "the module appeared without exactly one generation bump"
9992 );
9993 }
9994
9995 #[tokio::test]
10000 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
10001 let registry = Arc::new(Registry::default());
10002 let handler = ControlHandler::with_forwarding(
10003 Arc::clone(®istry),
10004 Arc::new(ForwardingTable::default()),
10005 );
10006 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
10007 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
10008 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
10009 drop(module_rx);
10010
10011 let replies = handler
10012 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
10013 .await
10014 .unwrap();
10015 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
10016 assert!(registry.get_module("aft").unwrap().is_none());
10017 assert!(registry
10018 .get_module_by_connection(module_ctx.connection_id)
10019 .unwrap()
10020 .is_none());
10021 assert_eq!(
10022 catalog_snapshot(&handler, &client_ctx, 301).await,
10023 (before, Vec::new())
10024 );
10025 }
10026
10027 async fn route_open_held_after_bind_send() -> (
10031 ControlHandler,
10032 Arc<tokio::sync::Barrier>,
10033 RouteCtx,
10034 mpsc::Receiver<crate::router::OutboundFrame>,
10035 ) {
10036 let hold = Arc::new(tokio::sync::Barrier::new(2));
10037 let handler = ControlHandler::with_forwarding(
10038 Arc::new(Registry::default()),
10039 Arc::new(ForwardingTable::default()),
10040 )
10041 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
10042 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
10043 hello_via_sink(
10044 &handler,
10045 &module_ctx,
10046 &mut module_rx,
10047 hello_frame("aft", PROTOCOL_VERSION, 7),
10048 )
10049 .await;
10050 (handler, hold, module_ctx, module_rx)
10051 }
10052
10053 #[tokio::test]
10060 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
10061 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10062 let client = ConnectionId::new(41);
10063 let (client_ctx, mut client_rx) = route_ctx(client);
10064
10065 let (open_task, bind) = relay_route_open(
10066 &handler,
10067 client,
10068 &client_ctx.egress,
10069 &mut module_rx,
10070 200,
10071 "aft",
10072 "acked-before-mark",
10073 )
10074 .await;
10075 let (module_channel, module_epoch) = route_bind_channel(&bind);
10076 hold.wait().await;
10078
10079 assert!(handler
10081 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10082 .await
10083 .unwrap()
10084 .is_empty());
10085 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
10086
10087 hold.wait().await;
10088 assert!(
10089 open_task.await.unwrap().is_empty(),
10090 "route.open answers accepted"
10091 );
10092
10093 match module_rx.try_recv() {
10096 Err(mpsc::error::TryRecvError::Empty) => {}
10097 Ok(outbound) => panic!(
10098 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
10099 outbound.frame.header.ty,
10100 outbound.frame.header.channel,
10101 outbound.frame.header.epoch
10102 ),
10103 Err(err) => panic!("module egress closed: {err}"),
10104 }
10105
10106 let router = Router::with_control_handler(Arc::new(handler.clone()));
10109 let request = Frame::build(
10110 FrameType::Request,
10111 Flags::new(true, Priority::Interactive, false),
10112 client_channel,
10113 client_epoch,
10114 201,
10115 b"first request".to_vec(),
10116 )
10117 .unwrap();
10118 router
10119 .route_for_connection(&client_ctx, request)
10120 .await
10121 .unwrap();
10122 let delivered = module_rx.try_recv().unwrap().frame;
10123 assert_eq!(delivered.header.ty, FrameType::Request);
10124 assert_eq!(
10125 (delivered.header.channel, delivered.header.epoch),
10126 (module_channel, module_epoch)
10127 );
10128 assert_eq!(delivered.body, b"first request");
10129 }
10130
10131 #[tokio::test]
10137 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10138 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10139 let client = ConnectionId::new(42);
10140 let (client_ctx, _client_rx) = route_ctx(client);
10141
10142 let (open_task, bind) = relay_route_open(
10143 &handler,
10144 client,
10145 &client_ctx.egress,
10146 &mut module_rx,
10147 300,
10148 "aft",
10149 "released-before-mark",
10150 )
10151 .await;
10152 let (module_channel, module_epoch) = route_bind_channel(&bind);
10153 hold.wait().await;
10154
10155 handler.cleanup_connection(client).unwrap();
10156 assert!(
10157 module_rx.try_recv().is_err(),
10158 "client teardown must not GOODBYE a bind route.open is still queueing"
10159 );
10160
10161 hold.wait().await;
10162 let response = open_task.await.unwrap();
10163 assert_eq!(response.len(), 1);
10164 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10165
10166 let goodbye = module_rx
10167 .try_recv()
10168 .expect("module is told to drop the binding the daemon abandoned")
10169 .frame;
10170 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10171 assert_eq!(
10172 (goodbye.header.channel, goodbye.header.epoch),
10173 (module_channel, module_epoch)
10174 );
10175 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10176 }
10177
10178 #[tokio::test]
10183 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
10184 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10185 let client = ConnectionId::new(43);
10186 let (client_ctx, client_rx) = route_ctx(client);
10187
10188 let (open_task, bind) = relay_route_open(
10189 &handler,
10190 client,
10191 &client_ctx.egress,
10192 &mut module_rx,
10193 400,
10194 "aft",
10195 "gone-client-before-mark",
10196 )
10197 .await;
10198 let (module_channel, module_epoch) = route_bind_channel(&bind);
10199 hold.wait().await;
10200
10201 drop(client_rx);
10202 assert!(client_ctx.egress.is_closed());
10203 handler
10204 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10205 .await
10206 .unwrap();
10207
10208 hold.wait().await;
10209 let response = open_task.await.unwrap();
10210 assert_eq!(response.len(), 1);
10211 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10212
10213 let goodbye = module_rx
10214 .try_recv()
10215 .expect("module is told to drop the binding it acked")
10216 .frame;
10217 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10218 assert_eq!(
10219 (goodbye.header.channel, goodbye.header.epoch),
10220 (module_channel, module_epoch)
10221 );
10222 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10223 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
10224 }
10225
10226 #[test]
10233 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
10234 let handler = ControlHandler::default();
10235 let module_connection = ConnectionId::new(30);
10236 let client_connection = ConnectionId::new(31);
10237
10238 handler
10239 .refuse_to_end_module_connection_for_a_client(
10240 module_connection,
10241 77,
10242 ForwardingError::ConnectionClosing {
10243 connection_id: client_connection,
10244 },
10245 )
10246 .expect("a closing client must never end the module connection");
10247
10248 assert!(matches!(
10249 handler.refuse_to_end_module_connection_for_a_client(
10250 module_connection,
10251 78,
10252 ForwardingError::ConnectionClosing {
10253 connection_id: module_connection,
10254 },
10255 ),
10256 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
10257 connection_id
10258 })) if connection_id == module_connection
10259 ));
10260 assert!(matches!(
10261 handler.refuse_to_end_module_connection_for_a_client(
10262 module_connection,
10263 79,
10264 ForwardingError::Poisoned,
10265 ),
10266 Err(RouterError::Forwarding(ForwardingError::Poisoned))
10267 ));
10268 assert!(matches!(
10269 handler.refuse_to_end_module_connection_for_a_client(
10270 module_connection,
10271 80,
10272 ForwardingError::StaleModuleEndpoint,
10273 ),
10274 Err(RouterError::Forwarding(
10275 ForwardingError::StaleModuleEndpoint
10276 ))
10277 ));
10278 }
10279
10280 #[tokio::test]
10293 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
10294 let registry = Arc::new(Registry::default());
10295 let forwarding = Arc::new(ForwardingTable::default());
10296 let supervisor = SupervisorHandle::new();
10297 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10298 let handler =
10299 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10300 .with_supervisor(supervisor);
10301
10302 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10303 hello_via_sink(
10304 &handler,
10305 &target_ctx,
10306 &mut target_rx,
10307 hello_frame("target", PROTOCOL_VERSION, 1),
10308 )
10309 .await;
10310
10311 let wrong_nonce = handler
10315 .handle_control_frame(
10316 &route_ctx(ConnectionId::new(91)).0,
10317 route_open_frame_with_admission_facts(
10318 20,
10319 "target",
10320 unique_project_root("admission-facts"),
10321 Some(subc_control::ConsumerIdentity {
10322 module_id: "fed".to_string(),
10323 launch_nonce: "not-the-real-nonce".to_string(),
10324 }),
10325 None,
10326 ),
10327 )
10328 .await
10329 .unwrap();
10330 assert_eq!(
10331 parse_error(&wrong_nonce[0])["code"],
10332 "bad_consumer_identity",
10333 "a mismatched launch nonce must be refused, not stamped Reserved"
10334 );
10335
10336 let never_spawned = handler
10340 .handle_control_frame(
10341 &route_ctx(ConnectionId::new(92)).0,
10342 route_open_frame_with_admission_facts(
10343 21,
10344 "target",
10345 unique_project_root("admission-facts"),
10346 Some(subc_control::ConsumerIdentity {
10347 module_id: "never-spawned".to_string(),
10348 launch_nonce: "any-nonce".to_string(),
10349 }),
10350 None,
10351 ),
10352 )
10353 .await
10354 .unwrap();
10355 assert_eq!(
10356 parse_error(&never_spawned[0])["code"],
10357 "bad_consumer_identity",
10358 "an unspawned module_id must be refused rather than accepted for lack of a record"
10359 );
10360 }
10361
10362 #[tokio::test]
10385 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
10386 let registry = Arc::new(Registry::default());
10387 let forwarding = Arc::new(ForwardingTable::default());
10388 let supervisor = SupervisorHandle::new();
10389 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10390 let handler =
10391 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10392 .with_supervisor(supervisor);
10393
10394 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
10395 hello_via_sink(
10396 &handler,
10397 &target_ctx,
10398 &mut target_rx,
10399 hello_frame("target", PROTOCOL_VERSION, 1),
10400 )
10401 .await;
10402
10403 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
10404 let route_handler = handler.clone();
10405 let route_task = tokio::spawn(async move {
10406 route_handler
10407 .handle_control_frame(
10408 &client_ctx,
10409 route_open_frame_with_admission_facts(
10410 30,
10411 "target",
10412 unique_project_root("admission-facts"),
10413 Some(subc_control::ConsumerIdentity {
10414 module_id: "fed".to_string(),
10415 launch_nonce: "fed-nonce".to_string(),
10416 }),
10417 None,
10418 ),
10419 )
10420 .await
10421 .unwrap()
10422 });
10423
10424 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10431 .await
10432 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
10433 .expect("module control channel closed before route.bind");
10434 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10435 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10436 panic!("expected route.bind")
10437 };
10438 assert_eq!(
10439 principal,
10440 Some(Principal::Reserved {
10441 module_id: "fed".to_string()
10442 }),
10443 "a correctly attested consumer must be stamped Reserved for its own id"
10444 );
10445
10446 handler
10447 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10448 .await
10449 .unwrap();
10450 assert!(route_task.await.unwrap().is_empty());
10451 assert!(
10452 matches!(
10453 serde_json::from_slice::<ClientControlResponse>(
10454 &client_rx.recv().await.unwrap().body
10455 )
10456 .unwrap(),
10457 ClientControlResponse::RouteOpen { .. }
10458 ),
10459 "the route must actually open, not merely avoid an error"
10460 );
10461 }
10462
10463 #[tokio::test(start_paused = true)]
10464 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10465 let registry = Arc::new(Registry::default());
10466 let forwarding = Arc::new(ForwardingTable::default());
10467 let supervisor = SupervisorHandle::new();
10468 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10469 let handler =
10470 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10471 .with_supervisor(supervisor);
10472
10473 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10474 hello_via_sink(
10475 &handler,
10476 &target_ctx,
10477 &mut target_rx,
10478 hello_frame("target", PROTOCOL_VERSION, 1),
10479 )
10480 .await;
10481
10482 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10483 let direct_handler = handler.clone();
10484 let direct_open = tokio::spawn(async move {
10485 direct_handler
10486 .handle_control_frame(
10487 &direct_ctx,
10488 route_open_frame(2, "target", unique_project_root("route-census-direct")),
10489 )
10490 .await
10491 .unwrap()
10492 });
10493 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10494 .await
10495 .expect("no direct route.bind within 5s")
10496 .expect("target control channel closed before direct route.bind");
10497 handler
10498 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10499 .await
10500 .unwrap();
10501 assert!(direct_open.await.unwrap().is_empty());
10502 let _ = direct_rx.recv().await.unwrap();
10503
10504 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10505 let reserved_handler = handler.clone();
10506 let reserved_open = tokio::spawn(async move {
10507 reserved_handler
10508 .handle_control_frame(
10509 &reserved_ctx,
10510 route_open_frame_with_admission_facts(
10511 3,
10512 "target",
10513 unique_project_root("admission-facts"),
10514 Some(ConsumerIdentity {
10515 module_id: "fed".to_string(),
10516 launch_nonce: "fed-nonce".to_string(),
10517 }),
10518 None,
10519 ),
10520 )
10521 .await
10522 .unwrap()
10523 });
10524 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10525 .await
10526 .expect("no reserved route.bind within 5s")
10527 .expect("target control channel closed before reserved route.bind");
10528 handler
10529 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10530 .await
10531 .unwrap();
10532 assert!(reserved_open.await.unwrap().is_empty());
10533 let _ = reserved_rx.recv().await.unwrap();
10534
10535 forwarding
10536 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10537 .unwrap();
10538 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
10539 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
10540 module_id: Some("target".to_string()),
10541 })
10542 .unwrap();
10543 let census_frame =
10544 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
10545 let response = handler
10546 .handle_control_frame(&census_ctx, census_frame)
10547 .await
10548 .unwrap()
10549 .pop()
10550 .unwrap();
10551 let actual: Value = serde_json::from_slice(&response.body).unwrap();
10552 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
10553 assert!(matches!(
10554 decoded,
10555 ClientControlResponse::SupervisorRoutes { .. }
10556 ));
10557 let routes = actual["modules"][0]["routes"].as_array().unwrap();
10558 assert_eq!(routes.len(), 2);
10559 assert!(routes.iter().all(|route| route["draining"] == true));
10560 assert!(
10563 routes.iter().all(|route| route["drain_reason"] == "reload"),
10564 "draining routes must name the drain's reason: {routes:?}"
10565 );
10566 assert!(routes.iter().any(|route| {
10567 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
10568 }));
10569 assert!(routes.iter().any(|route| {
10570 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
10571 }));
10572
10573 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10574 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
10575 if std::env::var_os("UPDATE_GOLDEN").is_some() {
10576 std::fs::write(
10577 &golden_path,
10578 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
10579 )
10580 .unwrap();
10581 }
10582 let expected: Value =
10583 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
10584 assert_eq!(actual, expected);
10585 }
10586
10587 async fn query_live_roots(
10588 handler: &ControlHandler,
10589 module_ctx: &RouteCtx,
10590 ) -> ModuleControlResponseToModule {
10591 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
10592 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
10593 let response = handler
10594 .handle_control_frame(module_ctx, frame)
10595 .await
10596 .unwrap()
10597 .pop()
10598 .unwrap();
10599 serde_json::from_slice(&response.body).unwrap()
10600 }
10601
10602 #[tokio::test(start_paused = true)]
10603 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
10604 let registry = Arc::new(Registry::default());
10605 let forwarding = Arc::new(ForwardingTable::default());
10606 let handler = ControlHandler::with_forwarding(registry, forwarding);
10607 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
10608 hello_via_sink(
10609 &handler,
10610 &target_ctx,
10611 &mut target_rx,
10612 hello_frame("target", PROTOCOL_VERSION, 1),
10613 )
10614 .await;
10615 let root = unique_project_root("live-roots-known");
10616 let path = ProjectRootId::from_path_allowing_missing(root.path())
10617 .unwrap()
10618 .as_path()
10619 .to_path_buf();
10620 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
10621 let open_handler = handler.clone();
10622 let opened = tokio::spawn(async move {
10623 open_handler
10624 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
10625 .await
10626 .unwrap()
10627 });
10628 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10629 .await
10630 .unwrap()
10631 .unwrap();
10632 handler
10633 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
10634 .await
10635 .unwrap();
10636 assert!(opened.await.unwrap().is_empty());
10637 let _ = client_rx.recv().await.unwrap();
10638
10639 let root = unique_project_root("live-roots-pending");
10640 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
10641 .unwrap()
10642 .as_path()
10643 .to_path_buf();
10644 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
10645 let open_handler = handler.clone();
10646 let pending = tokio::spawn(async move {
10647 open_handler
10648 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
10649 .await
10650 .unwrap()
10651 });
10652 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10653 .await
10654 .unwrap()
10655 .unwrap();
10656 let actual = query_live_roots(&handler, &target_ctx).await;
10657 let ModuleControlResponseToModule::LiveRoots {
10658 roots,
10659 unknown_root_bindings,
10660 total_bindings,
10661 } = actual
10662 else {
10663 panic!("expected live roots")
10664 };
10665 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
10666 assert_eq!(unknown_root_bindings, 0);
10667 assert_eq!(
10668 roots.len(),
10669 2,
10670 "root-known arm must retain each canonical root"
10671 );
10672 assert_eq!(
10673 total_bindings,
10674 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10675 );
10676 let counts = roots
10677 .iter()
10678 .map(|root| (root.project_root.clone(), root.bound, root.pending))
10679 .collect::<Vec<_>>();
10680 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
10681 expected.sort_by(|a, b| a.0.cmp(&b.0));
10682 assert_eq!(
10683 counts, expected,
10684 "roots must sort by path and count pending separately"
10685 );
10686 handler
10687 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
10688 .await
10689 .unwrap();
10690 assert!(pending.await.unwrap().is_empty());
10691 }
10692
10693 #[tokio::test(start_paused = true)]
10694 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
10695 let forwarding = Arc::new(ForwardingTable::default());
10696 let handler =
10697 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10698 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
10699 hello_via_sink(
10700 &handler,
10701 &target_ctx,
10702 &mut target_rx,
10703 hello_frame("target", PROTOCOL_VERSION, 1),
10704 )
10705 .await;
10706 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
10707 let pending = forwarding
10708 .begin_route_bind_relay_for_test(
10709 client_ctx.connection_id,
10710 client_ctx.egress.clone(),
10711 2,
10712 "target",
10713 )
10714 .unwrap();
10715 forwarding
10716 .complete_pending_relay(
10717 target_ctx.connection_id,
10718 pending.corr,
10719 RouteBindRelayOutcome::Accepted,
10720 )
10721 .unwrap();
10722 let actual = query_live_roots(&handler, &target_ctx).await;
10723 let ModuleControlResponseToModule::LiveRoots {
10724 roots,
10725 unknown_root_bindings,
10726 total_bindings,
10727 } = actual
10728 else {
10729 panic!("expected live roots")
10730 };
10731 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
10732 assert_eq!(
10733 unknown_root_bindings, 1,
10734 "unknown-root arm must not read as no bindings"
10735 );
10736 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
10737 assert_eq!(
10738 total_bindings,
10739 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10740 );
10741 }
10742
10743 #[tokio::test(start_paused = true)]
10751 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
10752 let forwarding = Arc::new(ForwardingTable::default());
10753 let handler =
10754 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10755 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
10756 let replies = handler
10757 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
10758 .await
10759 .unwrap();
10760 let queued_by_hello = module_rx.len();
10761
10762 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
10763 let open_handler = handler.clone();
10764 let open = tokio::spawn(async move {
10765 open_handler
10766 .handle_control_frame(
10767 &client_ctx,
10768 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
10769 )
10770 .await
10771 .unwrap()
10772 });
10773 let mut spins = 0;
10775 while module_rx.len() == queued_by_hello {
10776 spins += 1;
10777 assert!(spins < 10_000, "route.open never queued a route.bind");
10778 tokio::task::yield_now().await;
10779 }
10780
10781 for reply in replies {
10783 module_ctx.egress.send(reply).await.unwrap();
10784 }
10785
10786 let first = module_rx.recv().await.unwrap().frame;
10787 assert_eq!(
10788 first.header.ty,
10789 FrameType::HelloAck,
10790 "the first frame a registering module reads must be its HELLO_ACK"
10791 );
10792 assert_eq!(first.header.corr, 7);
10793 let second = module_rx.recv().await.unwrap().frame;
10794 assert_eq!(second.header.ty, FrameType::Request);
10795 assert!(
10796 matches!(
10797 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
10798 ModuleControlRequest::RouteBind { .. }
10799 ),
10800 "the route.bind follows the ack"
10801 );
10802 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
10803
10804 handler
10805 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
10806 .await
10807 .unwrap();
10808 assert!(open.await.unwrap().is_empty());
10809 let _ = client_rx.recv().await.unwrap();
10810 }
10811
10812 #[tokio::test(start_paused = true)]
10813 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
10814 let handler = ControlHandler::with_forwarding(
10815 Arc::new(Registry::default()),
10816 Arc::new(ForwardingTable::default()),
10817 );
10818 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
10819 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
10820 hello_via_sink(
10821 &handler,
10822 &first_ctx,
10823 &mut first_rx,
10824 hello_frame("first", PROTOCOL_VERSION, 1),
10825 )
10826 .await;
10827 hello_via_sink(
10828 &handler,
10829 &second_ctx,
10830 &mut second_rx,
10831 hello_frame("second", PROTOCOL_VERSION, 2),
10832 )
10833 .await;
10834 let root = unique_project_root("second-only");
10835 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
10836 let cloned = handler.clone();
10837 let open = tokio::spawn(async move {
10838 cloned
10839 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
10840 .await
10841 .unwrap()
10842 });
10843 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
10844 .await
10845 .unwrap()
10846 .unwrap();
10847 let first = query_live_roots(&handler, &first_ctx).await;
10848 let second = query_live_roots(&handler, &second_ctx).await;
10849 assert!(
10850 matches!(
10851 first,
10852 ModuleControlResponseToModule::LiveRoots {
10853 total_bindings: 0,
10854 ..
10855 }
10856 ),
10857 "cross-module scope must not expose another module's roots"
10858 );
10859 assert!(
10860 matches!(
10861 second,
10862 ModuleControlResponseToModule::LiveRoots {
10863 total_bindings: 1,
10864 ..
10865 }
10866 ),
10867 "second module must see its pending route"
10868 );
10869 handler
10870 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
10871 .await
10872 .unwrap();
10873 assert!(open.await.unwrap().is_empty());
10874 }
10875
10876 #[tokio::test(start_paused = true)]
10877 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
10878 let handler = ControlHandler::with_forwarding(
10879 Arc::new(Registry::default()),
10880 Arc::new(ForwardingTable::default()),
10881 );
10882 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
10883 hello_via_sink(
10884 &handler,
10885 &target_ctx,
10886 &mut target_rx,
10887 hello_frame("target", PROTOCOL_VERSION, 1),
10888 )
10889 .await;
10890 let actual = query_live_roots(&handler, &target_ctx).await;
10891 let ModuleControlResponseToModule::LiveRoots {
10892 roots,
10893 unknown_root_bindings,
10894 total_bindings,
10895 } = actual
10896 else {
10897 panic!("expected live roots")
10898 };
10899 assert!(roots.is_empty());
10900 assert_eq!(unknown_root_bindings, 0);
10901 assert_eq!(total_bindings, 0);
10902 assert_eq!(
10903 total_bindings,
10904 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10905 );
10906 }
10907
10908 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
10915 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10916 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
10917 let text = std::fs::read_to_string(&path)
10918 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
10919 let vectors: Vec<(String, Value)> = text
10920 .lines()
10921 .filter(|line| !line.trim().is_empty())
10922 .map(|line| {
10923 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
10924 let id = entry["corpus_id"]
10925 .as_str()
10926 .expect("every vector carries a corpus_id")
10927 .to_string();
10928 (id, entry["package"].clone())
10929 })
10930 .collect();
10931 assert_eq!(
10935 vectors.len(),
10936 3,
10937 "vendored fed corpus changed size; re-sync from subc-federation"
10938 );
10939
10940 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
10955 let richest = vectors
10956 .iter()
10957 .filter_map(|(_, package)| package.as_object())
10958 .map(|object| {
10959 object
10960 .keys()
10961 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
10962 .count()
10963 })
10964 .max()
10965 .unwrap_or(0);
10966 assert!(
10967 richest >= 2,
10968 "vendored corpus no longer carries a package with unmodelled fields, \
10969 so the relay test can no longer distinguish a verbatim relay from a lossy one"
10970 );
10971
10972 vectors
10973 }
10974
10975 #[tokio::test]
10983 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
10984 for (corpus_id, package) in fed_admission_facts_vectors() {
10985 let registry = Arc::new(Registry::default());
10986 let forwarding = Arc::new(ForwardingTable::default());
10987 let supervisor = SupervisorHandle::new();
10988 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10989 let handler =
10990 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10991 .with_supervisor(supervisor)
10992 .with_admission_facts_config(
10993 Some("fed".to_string()),
10994 Some(vec!["target".to_string()]),
10995 );
10996
10997 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10998 hello_via_sink(
10999 &handler,
11000 &target_ctx,
11001 &mut target_rx,
11002 hello_frame("target", PROTOCOL_VERSION, 1),
11003 )
11004 .await;
11005
11006 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
11007 let route_handler = handler.clone();
11008 let expected = package.clone();
11009 let route_task = tokio::spawn(async move {
11010 route_handler
11011 .handle_control_frame(
11012 &client_ctx,
11013 route_open_frame_with_admission_facts(
11014 20,
11015 "target",
11016 unique_project_root("admission-facts"),
11017 Some(subc_control::ConsumerIdentity {
11018 module_id: "fed".to_string(),
11019 launch_nonce: "fed-nonce".to_string(),
11020 }),
11021 Some(package),
11022 ),
11023 )
11024 .await
11025 .unwrap()
11026 });
11027
11028 let bind_frame = target_rx.recv().await.unwrap();
11029 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11030 let ModuleControlRequest::RouteBind {
11031 admission_facts, ..
11032 } = bind
11033 else {
11034 panic!("{corpus_id}: expected route.bind")
11035 };
11036 assert_eq!(
11037 admission_facts,
11038 Some(expected),
11039 "{corpus_id}: relay must not add, drop or reshape any field"
11040 );
11041
11042 handler
11043 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11044 .await
11045 .unwrap();
11046 route_task.await.unwrap();
11047 }
11048 }
11049
11050 #[tokio::test]
11051 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
11052 let registry = Arc::new(Registry::default());
11053 let forwarding = Arc::new(ForwardingTable::default());
11054 let supervisor = SupervisorHandle::new();
11055 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11056 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
11057 let handler =
11058 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11059 .with_supervisor(supervisor)
11060 .with_admission_facts_config(
11061 Some("fed".to_string()),
11062 Some(vec!["target".to_string()]),
11063 );
11064
11065 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
11066 hello_via_sink(
11067 &handler,
11068 &target_ctx,
11069 &mut target_rx,
11070 hello_frame("target", PROTOCOL_VERSION, 1),
11071 )
11072 .await;
11073 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
11074 hello_via_sink(
11075 &handler,
11076 &other_ctx,
11077 &mut other_rx,
11078 hello_frame("other", PROTOCOL_VERSION, 2),
11079 )
11080 .await;
11081
11082 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
11083 let expected_facts = facts.clone();
11084 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
11085 let route_handler = handler.clone();
11086 let route_task = tokio::spawn(async move {
11087 route_handler
11088 .handle_control_frame(
11089 &client_ctx,
11090 route_open_frame_with_admission_facts(
11091 10,
11092 "target",
11093 unique_project_root("admission-facts"),
11094 Some(subc_control::ConsumerIdentity {
11095 module_id: "fed".to_string(),
11096 launch_nonce: "fed-nonce".to_string(),
11097 }),
11098 Some(facts.clone()),
11099 ),
11100 )
11101 .await
11102 .unwrap()
11103 });
11104 let bind_frame = target_rx.recv().await.unwrap();
11105 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11106 let ModuleControlRequest::RouteBind {
11107 admission_facts, ..
11108 } = bind
11109 else {
11110 panic!("expected route.bind")
11111 };
11112 assert_eq!(admission_facts, Some(expected_facts));
11113 handler
11114 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11115 .await
11116 .unwrap();
11117 assert!(route_task.await.unwrap().is_empty());
11118 assert!(matches!(
11119 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
11120 .unwrap(),
11121 ClientControlResponse::RouteOpen { .. }
11122 ));
11123
11124 let direct = handler
11125 .handle_control_frame(
11126 &route_ctx(ConnectionId::new(73)).0,
11127 route_open_frame_with_admission_facts(
11128 11,
11129 "target",
11130 unique_project_root("admission-facts"),
11131 None,
11132 Some(json!({"x": 1})),
11133 ),
11134 )
11135 .await
11136 .unwrap();
11137 assert_eq!(
11138 parse_error(&direct[0])["code"],
11139 "admission_facts_not_permitted"
11140 );
11141
11142 let different_reserved = handler
11143 .handle_control_frame(
11144 &route_ctx(ConnectionId::new(77)).0,
11145 route_open_frame_with_admission_facts(
11146 15,
11147 "target",
11148 unique_project_root("admission-facts"),
11149 Some(subc_control::ConsumerIdentity {
11150 module_id: "other".to_string(),
11151 launch_nonce: "other-nonce".to_string(),
11152 }),
11153 Some(json!({"x": 1})),
11154 ),
11155 )
11156 .await
11157 .unwrap();
11158 assert_eq!(
11159 parse_error(&different_reserved[0])["code"],
11160 "admission_facts_not_permitted"
11161 );
11162
11163 let other_target = handler
11164 .handle_control_frame(
11165 &route_ctx(ConnectionId::new(74)).0,
11166 route_open_frame_with_admission_facts(
11167 12,
11168 "other",
11169 unique_project_root("admission-facts"),
11170 Some(subc_control::ConsumerIdentity {
11171 module_id: "fed".to_string(),
11172 launch_nonce: "fed-nonce".to_string(),
11173 }),
11174 Some(json!({"x": 1})),
11175 ),
11176 )
11177 .await
11178 .unwrap();
11179 assert_eq!(
11180 parse_error(&other_target[0])["code"],
11181 "admission_facts_target_not_allowed"
11182 );
11183
11184 let nonexistent = handler
11185 .handle_control_frame(
11186 &route_ctx(ConnectionId::new(75)).0,
11187 route_open_frame_with_admission_facts(
11188 13,
11189 "missing",
11190 unique_project_root("admission-facts"),
11191 None,
11192 Some(json!({"x": 1})),
11193 ),
11194 )
11195 .await
11196 .unwrap();
11197 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
11198
11199 let described = handler
11200 .handle_control_frame(
11201 &route_ctx(ConnectionId::new(76)).0,
11202 Frame::build(
11203 FrameType::Request,
11204 control_flags(),
11205 0,
11206 0,
11207 14,
11208 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11209 )
11210 .unwrap(),
11211 )
11212 .await
11213 .unwrap();
11214 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11215 serde_json::from_slice(&described[0].body).unwrap()
11216 else {
11217 panic!("expected server.describe response")
11218 };
11219 assert!(capabilities
11220 .iter()
11221 .any(|cap| cap == "admission_facts_relay_v1"));
11222 }
11223
11224 #[tokio::test]
11225 async fn admission_facts_without_configured_carrier_are_rejected() {
11226 let registry = Arc::new(Registry::default());
11227 let forwarding = Arc::new(ForwardingTable::default());
11228 let handler = ControlHandler::with_forwarding(registry, forwarding);
11229 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
11230 hello_via_sink(
11231 &handler,
11232 &target_ctx,
11233 &mut target_rx,
11234 hello_frame("target", PROTOCOL_VERSION, 1),
11235 )
11236 .await;
11237
11238 let responses = handler
11239 .handle_control_frame(
11240 &route_ctx(ConnectionId::new(79)).0,
11241 route_open_frame_with_admission_facts(
11242 16,
11243 "target",
11244 unique_project_root("admission-facts"),
11245 None,
11246 Some(json!({"x": 1})),
11247 ),
11248 )
11249 .await
11250 .unwrap();
11251 assert_eq!(
11252 parse_error(&responses[0])["code"],
11253 "admission_facts_not_permitted"
11254 );
11255 }
11256
11257 #[tokio::test]
11258 async fn route_open_relays_consumer_capabilities_verbatim() {
11259 let registry = Arc::new(Registry::default());
11260 let forwarding = Arc::new(ForwardingTable::default());
11261 let handler =
11262 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11263 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
11264 hello_via_sink(
11265 &handler,
11266 &module_ctx,
11267 &mut module_rx,
11268 hello_frame("aft", PROTOCOL_VERSION, 7),
11269 )
11270 .await;
11271
11272 let expected = vec!["elicitation".to_string(), "roots".to_string()];
11273 let expected_for_request = expected.clone();
11274 let project_root = unique_project_root("consumer-capabilities-present");
11275 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
11276 let route_handler = handler.clone();
11277 let route_task = tokio::spawn(async move {
11278 route_handler
11279 .handle_control_frame(
11280 &client_ctx,
11281 route_open_frame_with_consumer_capabilities(
11282 401,
11283 "aft",
11284 project_root,
11285 Some(expected_for_request),
11286 ),
11287 )
11288 .await
11289 .unwrap()
11290 });
11291 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11292 .await
11293 .unwrap()
11294 .unwrap();
11295 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11296 let ModuleControlRequest::RouteBind {
11297 consumer_capabilities,
11298 ..
11299 } = bind
11300 else {
11301 panic!("expected route.bind request, got {bind:?}");
11302 };
11303 assert_eq!(consumer_capabilities, Some(expected.clone()));
11304
11305 handler
11306 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11307 .await
11308 .unwrap();
11309 let route_response = route_task.await.unwrap();
11310 assert!(route_response.is_empty());
11311 let published = client_rx.recv().await.unwrap();
11312 assert!(matches!(
11313 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11314 ClientControlResponse::RouteOpen { .. }
11315 ));
11316 }
11317
11318 #[tokio::test]
11319 async fn route_open_without_consumer_capabilities_relays_none() {
11320 let registry = Arc::new(Registry::default());
11321 let forwarding = Arc::new(ForwardingTable::default());
11322 let handler =
11323 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11324 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
11325 hello_via_sink(
11326 &handler,
11327 &module_ctx,
11328 &mut module_rx,
11329 hello_frame("aft", PROTOCOL_VERSION, 7),
11330 )
11331 .await;
11332
11333 let project_root = unique_project_root("consumer-capabilities-absent");
11334 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
11335 let route_handler = handler.clone();
11336 let route_task = tokio::spawn(async move {
11337 route_handler
11338 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
11339 .await
11340 .unwrap()
11341 });
11342 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11343 .await
11344 .unwrap()
11345 .unwrap();
11346 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11347 let ModuleControlRequest::RouteBind {
11348 consumer_capabilities,
11349 ..
11350 } = bind
11351 else {
11352 panic!("expected route.bind request, got {bind:?}");
11353 };
11354 assert_eq!(consumer_capabilities, None);
11355
11356 handler
11357 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11358 .await
11359 .unwrap();
11360 let route_response = route_task.await.unwrap();
11361 assert!(route_response.is_empty());
11362 let published = client_rx.recv().await.unwrap();
11363 assert!(matches!(
11364 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11365 ClientControlResponse::RouteOpen { .. }
11366 ));
11367 }
11368
11369 async fn bind_role_versions_for(
11373 sent: Option<BTreeMap<String, String>>,
11374 connection: u64,
11375 ) -> Option<BTreeMap<String, String>> {
11376 let registry = Arc::new(Registry::default());
11377 let forwarding = Arc::new(ForwardingTable::default());
11378 let handler =
11379 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11380 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
11381 hello_via_sink(
11382 &handler,
11383 &module_ctx,
11384 &mut module_rx,
11385 hello_frame("aft", PROTOCOL_VERSION, 7),
11386 )
11387 .await;
11388 let project_root = unique_project_root("role-versions");
11389 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
11390 let route_handler = handler.clone();
11391 let route_task = tokio::spawn(async move {
11392 route_handler
11393 .handle_control_frame(
11394 &client_ctx,
11395 route_open_frame_with_role_versions(403, "aft", project_root, sent),
11396 )
11397 .await
11398 .unwrap()
11399 });
11400 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11401 .await
11402 .expect("a well-formed route.open reaches the module as a bind")
11403 .unwrap();
11404 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11405 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
11406 panic!("expected route.bind request, got {bind:?}");
11407 };
11408 handler
11409 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11410 .await
11411 .unwrap();
11412 assert!(route_task.await.unwrap().is_empty());
11413 let published = client_rx.recv().await.unwrap();
11414 assert!(matches!(
11415 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11416 ClientControlResponse::RouteOpen { .. }
11417 ));
11418 role_versions
11419 }
11420
11421 #[tokio::test]
11422 async fn route_open_relays_role_versions_verbatim() {
11423 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
11424 assert_eq!(
11425 bind_role_versions_for(Some(sent.clone()), 141).await,
11426 Some(sent)
11427 );
11428 }
11429
11430 #[tokio::test]
11433 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
11434 assert_eq!(bind_role_versions_for(None, 143).await, None);
11435 assert_eq!(
11436 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
11437 None
11438 );
11439 }
11440
11441 #[tokio::test]
11442 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
11443 let registry = Arc::new(Registry::default());
11444 let forwarding = Arc::new(ForwardingTable::default());
11445 let handler =
11446 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11447 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
11448 hello_via_sink(
11449 &handler,
11450 &module_ctx,
11451 &mut module_rx,
11452 hello_frame("aft", PROTOCOL_VERSION, 7),
11453 )
11454 .await;
11455
11456 let nine: BTreeMap<String, String> = (0..9)
11457 .map(|index| (format!("role-{index}"), "v1".to_string()))
11458 .collect();
11459 for (label, malformed) in [
11460 (
11461 "invalid role name",
11462 role_versions(&[("Tool_Provider", "v1")]),
11463 ),
11464 ("invalid version", role_versions(&[("tool-provider", "v0")])),
11465 ("nine entries", nine),
11466 ] {
11467 let responses = tokio::time::timeout(
11470 Duration::from_secs(1),
11471 handler.handle_control_frame(
11472 &route_ctx(ConnectionId::new(148)).0,
11473 route_open_frame_with_role_versions(
11474 404,
11475 "aft",
11476 unique_project_root("role-versions-malformed"),
11477 Some(malformed),
11478 ),
11479 ),
11480 )
11481 .await
11482 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11483 .unwrap();
11484 assert_eq!(responses.len(), 1, "{label}");
11485 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11486 let error = parse_error(&responses[0]);
11487 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11488 assert_eq!(
11489 error["detail"]["field"], "role_versions",
11490 "{label}: {error}"
11491 );
11492 assert!(
11493 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11494 "{label}: a malformed declaration is terminal"
11495 );
11496 assert!(
11497 module_rx.try_recv().is_err(),
11498 "{label}: the module must never see a bind"
11499 );
11500 }
11501 }
11502
11503 #[tokio::test]
11507 async fn route_role_versions_capability_is_advertised() {
11508 let handler = ControlHandler::new(Arc::new(Registry::default()));
11509 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11510 let ack = hello_via_sink(
11511 &handler,
11512 &ctx,
11513 &mut rx,
11514 hello_frame("m", PROTOCOL_VERSION, 1),
11515 )
11516 .await;
11517 let ack = parse_ack(&ack);
11518 assert!(
11519 ack.subc_capabilities
11520 .iter()
11521 .any(|c| c == "route-role-versions/v1"),
11522 "{:?}",
11523 ack.subc_capabilities
11524 );
11525
11526 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11527 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11528 let reply = handler
11529 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11530 .await
11531 .unwrap()
11532 .pop()
11533 .unwrap();
11534 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11535 serde_json::from_slice(&reply.body).unwrap()
11536 else {
11537 panic!("not a server.describe reply");
11538 };
11539 assert!(
11540 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
11541 "{capabilities:?}"
11542 );
11543 }
11544
11545 #[tokio::test]
11549 async fn catalog_events_capability_is_advertised() {
11550 let handler = ControlHandler::new(Arc::new(Registry::default()));
11551 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11552 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11553 let reply = handler
11554 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
11555 .await
11556 .unwrap()
11557 .pop()
11558 .unwrap();
11559 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11560 serde_json::from_slice(&reply.body).unwrap()
11561 else {
11562 panic!("not a server.describe reply");
11563 };
11564 assert!(
11565 capabilities
11566 .iter()
11567 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
11568 "{capabilities:?}"
11569 );
11570 }
11571
11572 #[tokio::test]
11573 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
11574 let handler = ControlHandler::new(Arc::new(Registry::default()));
11575 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
11576 let ack = hello_via_sink(
11577 &handler,
11578 &module,
11579 &mut module_rx,
11580 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
11581 )
11582 .await;
11583 let ack = parse_ack(&ack);
11584 assert!(
11585 ack.subc_capabilities
11586 .iter()
11587 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11588 "{:?}",
11589 ack.subc_capabilities
11590 );
11591 assert!(ack
11592 .subc_capabilities
11593 .iter()
11594 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11595
11596 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
11597 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11598 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11599 let reply = handler
11600 .handle_control_frame(&client, frame)
11601 .await
11602 .unwrap()
11603 .pop()
11604 .unwrap();
11605 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11606 serde_json::from_slice(&reply.body).unwrap()
11607 else {
11608 panic!("not a server.describe reply");
11609 };
11610 assert!(
11611 capabilities
11612 .iter()
11613 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11614 "{capabilities:?}"
11615 );
11616 assert!(capabilities
11617 .iter()
11618 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11619 }
11620
11621 #[tokio::test]
11622 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
11623 let handler = ControlHandler::new(Arc::new(Registry::default()));
11624 let (module, mut rx) = route_ctx(ConnectionId::new(156));
11625 let ack = hello_via_sink(
11626 &handler,
11627 &module,
11628 &mut rx,
11629 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
11630 )
11631 .await;
11632 assert!(parse_ack(&ack)
11633 .subc_capabilities
11634 .iter()
11635 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
11636 assert!(parse_ack(&ack)
11637 .subc_capabilities
11638 .iter()
11639 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
11640 let (client, _rx) = route_ctx(ConnectionId::new(157));
11641 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11642 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11643 let reply = handler
11644 .handle_control_frame(&client, frame)
11645 .await
11646 .unwrap()
11647 .pop()
11648 .unwrap();
11649 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11650 serde_json::from_slice(&reply.body).unwrap()
11651 else {
11652 panic!("server.describe reply")
11653 };
11654 assert!(capabilities
11655 .iter()
11656 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
11657 assert!(capabilities
11658 .iter()
11659 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
11660 }
11661
11662 #[tokio::test]
11663 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
11664 let handler = ControlHandler::new(Arc::new(Registry::default()));
11665 let (module, mut rx) = route_ctx(ConnectionId::new(154));
11666 let ack = hello_via_sink(
11667 &handler,
11668 &module,
11669 &mut rx,
11670 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
11671 )
11672 .await;
11673 assert!(parse_ack(&ack)
11674 .subc_capabilities
11675 .iter()
11676 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11677 let (client, _rx) = route_ctx(ConnectionId::new(155));
11678 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11679 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11680 let reply = handler
11681 .handle_control_frame(&client, frame)
11682 .await
11683 .unwrap()
11684 .pop()
11685 .unwrap();
11686 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11687 serde_json::from_slice(&reply.body).unwrap()
11688 else {
11689 panic!("not a server.describe reply")
11690 };
11691 assert!(capabilities
11692 .iter()
11693 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11694 }
11695
11696 #[tokio::test]
11697 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
11698 let registry = Arc::new(Registry::default());
11699 let forwarding = Arc::new(ForwardingTable::default());
11700 let handler =
11701 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11702 .with_health_probe_timeout(Duration::from_secs(5));
11703 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
11704 hello_via_sink(
11705 &handler,
11706 &module_ctx,
11707 &mut module_rx,
11708 non_routable_hello_frame_with_control_ops(
11709 "mcp",
11710 300,
11711 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11712 ),
11713 )
11714 .await;
11715 assert!(registry
11716 .get_module("mcp")
11717 .unwrap()
11718 .unwrap()
11719 .manifest
11720 .provides
11721 .is_empty());
11722
11723 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
11724 let route_response = handler
11725 .handle_control_frame(
11726 &route_client_ctx,
11727 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
11728 )
11729 .await
11730 .unwrap();
11731 assert_eq!(route_response[0].header.ty, FrameType::Error);
11732 assert_eq!(
11733 parse_error(&route_response[0])["code"],
11734 "target_unavailable"
11735 );
11736 assert!(parse_error(&route_response[0])["message"]
11737 .as_str()
11738 .unwrap()
11739 .contains("does not provide the requested target"));
11740 assert!(module_rx.try_recv().is_err());
11741
11742 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
11743 let health_handler = handler.clone();
11744 let health_task = tokio::spawn(async move {
11745 health_handler
11746 .handle_control_frame(
11747 &health_client_ctx,
11748 supervisor_health_probe_frame(302, "mcp"),
11749 )
11750 .await
11751 .unwrap()
11752 });
11753 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11754 .await
11755 .unwrap()
11756 .unwrap();
11757 assert_eq!(
11758 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11759 ModuleControlRequest::HealthCheck {}
11760 );
11761 handler
11762 .handle_control_frame(
11763 &module_ctx,
11764 health_response(health_frame.header.corr, HealthStatus::Ok),
11765 )
11766 .await
11767 .unwrap();
11768 let health_response = health_task.await.unwrap();
11769 assert_eq!(health_response[0].header.ty, FrameType::Response);
11770 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11771 ClientControlResponse::SupervisorHealthProbe {
11772 module_id, status, ..
11773 } => {
11774 assert_eq!(module_id, "mcp");
11775 assert_eq!(status, HealthStatus::Ok);
11776 }
11777 other => panic!("unexpected health response: {other:?}"),
11778 }
11779
11780 forwarding
11785 .cleanup_connection(module_ctx.connection_id)
11786 .unwrap();
11787 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
11788 let cleanup_response = tokio::time::timeout(
11789 Duration::from_millis(200),
11790 handler.handle_control_frame(
11791 &cleanup_probe_ctx,
11792 supervisor_health_probe_frame(303, "mcp"),
11793 ),
11794 )
11795 .await
11796 .expect("probe should fail immediately when the control lane is gone")
11797 .unwrap();
11798 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
11799 assert_eq!(
11800 parse_error(&cleanup_response[0])["code"],
11801 "target_unavailable"
11802 );
11803 assert!(parse_error(&cleanup_response[0])["message"]
11804 .as_str()
11805 .unwrap()
11806 .contains("no module connection"));
11807
11808 handler
11809 .cleanup_connection(module_ctx.connection_id)
11810 .unwrap();
11811 }
11812
11813 #[tokio::test]
11814 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
11815 let registry = Arc::new(Registry::default());
11816 let supervisor_handle = SupervisorHandle::new();
11817 let supervisor =
11818 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11819 .with_handle(supervisor_handle.clone())
11820 .with_connection_file_path(
11821 std::env::temp_dir()
11822 .join(format!("subc-route-open-warming-{}", std::process::id())),
11823 );
11824 let module = supervisor
11825 .supervise_configured(
11826 ModuleSpec {
11827 module_id: "warming".to_string(),
11828 program: fake_aft_stub_path(),
11829 args: Vec::new(),
11830 env: Vec::new(),
11831 reserved: false,
11832 reserved_prefixes: Vec::new(),
11833 protocol: ModuleProtocol::Subc,
11834 overlap: Default::default(),
11835 },
11836 true,
11837 )
11838 .unwrap();
11839 assert_eq!(module.state().unwrap(), ModuleState::Running);
11840
11841 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11842 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
11843 let response = handler
11844 .handle_control_frame(
11845 &ctx,
11846 route_open_frame(304, "warming", unique_project_root("warming")),
11847 )
11848 .await
11849 .unwrap();
11850 module.stop().await.unwrap();
11851
11852 assert_eq!(response[0].header.ty, FrameType::Error);
11853 let error = parse_error(&response[0]);
11854 assert_eq!(error["code"], "module_warming");
11855 assert!(error["message"]
11856 .as_str()
11857 .unwrap()
11858 .contains("state=running, enabled=true, live=false"));
11859 }
11860
11861 #[test]
11862 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
11863 let handler = ControlHandler::new(Arc::new(Registry::default()));
11864 let capture = EventCapture::default();
11865 let _subscriber =
11866 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11867 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
11868 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
11869 let pending = (0..limit).collect::<Vec<_>>();
11870 let response = handler
11871 .route_open_capacity_refusal(
11872 &ctx,
11873 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
11874 "busy",
11875 pending.len(),
11876 limit,
11877 )
11878 .unwrap();
11879 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11880 let event = capture
11881 .events()
11882 .into_iter()
11883 .find(|event| {
11884 event.target == "control"
11885 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
11886 })
11887 .expect("connection admission refusal event");
11888 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11889 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11890 }
11891
11892 #[test]
11893 fn route_open_target_cap_logs_admission_reason_and_capacity() {
11894 let handler = ControlHandler::new(Arc::new(Registry::default()));
11895 let capture = EventCapture::default();
11896 let _subscriber =
11897 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11898 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
11899 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
11900 let guards = (0..limit)
11901 .map(|_| {
11902 handler
11903 .route_bind_concurrency
11904 .try_admit("busy", limit)
11905 .unwrap()
11906 })
11907 .collect::<Vec<_>>();
11908 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
11909 Err(in_flight) => in_flight,
11910 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
11911 };
11912 let response = handler
11913 .route_open_target_capacity_refusal(
11914 &ctx,
11915 &route_open_frame(397, "busy", unique_project_root("target-cap")),
11916 "busy",
11917 in_flight,
11918 )
11919 .unwrap();
11920 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11921 let event = capture
11922 .events()
11923 .into_iter()
11924 .find(|event| {
11925 event.target == "control"
11926 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
11927 })
11928 .expect("target admission refusal event");
11929 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11930 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11931 drop(guards);
11932 }
11933
11934 #[tokio::test(flavor = "current_thread")]
11935 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
11936 let handler = ControlHandler::new(Arc::new(Registry::default()));
11937 let capture = EventCapture::default();
11938 let _subscriber =
11939 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11940 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
11941 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11942
11943 let _ = handler
11944 .handle_client_control_request(
11945 &ctx,
11946 frame,
11947 ClientControlRequest::SupervisorSetEnabled {
11948 module_id: "broca".to_string(),
11949 enabled: false,
11950 },
11951 &mut None,
11952 )
11953 .await
11954 .unwrap();
11955
11956 let events = capture
11957 .events()
11958 .into_iter()
11959 .filter(|event| {
11960 event.target == "control"
11961 && event.fields.get("message").map(String::as_str)
11962 == Some("supervisor request received")
11963 })
11964 .collect::<Vec<_>>();
11965 assert_eq!(events.len(), 1, "one supervisor request log line");
11966 let event = &events[0];
11967 assert_eq!(event.level, tracing::Level::INFO);
11968 assert_eq!(
11969 event.fields.get("op"),
11970 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
11971 );
11972 assert_eq!(
11973 event.fields.get("module_id"),
11974 Some(&"\"broca\"".to_string())
11975 );
11976 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
11977 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
11978 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
11979 }
11980
11981 #[tokio::test(flavor = "current_thread")]
11982 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
11983 let registry = Arc::new(Registry::default());
11984 let module_connection = ConnectionId::new(303);
11985 registry
11986 .register_with_control_ops(
11987 manifest("aft", PROTOCOL_VERSION),
11988 PROTOCOL_VERSION,
11989 module_connection,
11990 module_baseline_control_ops(),
11991 )
11992 .unwrap();
11993 let handler = ControlHandler::new(registry);
11994 let capture = EventCapture::default();
11995 let _subscriber =
11996 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11997 let (ctx, _rx) = route_ctx(module_connection);
11998 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11999
12000 let _ = handler
12001 .handle_client_control_request(
12002 &ctx,
12003 frame,
12004 ClientControlRequest::SupervisorSetEnabled {
12005 module_id: "broca".to_string(),
12006 enabled: false,
12007 },
12008 &mut None,
12009 )
12010 .await
12011 .unwrap();
12012
12013 let callers = capture
12014 .events()
12015 .into_iter()
12016 .filter(|event| {
12017 event.fields.get("message").map(String::as_str)
12018 == Some("supervisor request received")
12019 })
12020 .map(|event| event.fields.get("caller").cloned())
12021 .collect::<Vec<_>>();
12022 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
12023 }
12024
12025 #[tokio::test(flavor = "current_thread")]
12026 async fn supervisor_list_does_not_log_request_received() {
12027 let handler = ControlHandler::new(Arc::new(Registry::default()));
12028 let capture = EventCapture::default();
12029 let _subscriber =
12030 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12031 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
12032 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
12033
12034 handler
12035 .handle_client_control_request(
12036 &ctx,
12037 frame,
12038 ClientControlRequest::SupervisorList {},
12039 &mut None,
12040 )
12041 .await
12042 .unwrap();
12043
12044 assert!(capture.events().into_iter().all(|event| {
12045 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12046 }));
12047 }
12048
12049 #[tokio::test(flavor = "current_thread")]
12050 async fn supervisor_rescan_preview_does_not_log_request_received() {
12051 let handler = ControlHandler::new(Arc::new(Registry::default()));
12052 let capture = EventCapture::default();
12053 let _subscriber =
12054 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12055 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
12056 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
12057
12058 handler
12059 .handle_client_control_request(
12060 &ctx,
12061 frame,
12062 ClientControlRequest::SupervisorRescan { preview: true },
12063 &mut None,
12064 )
12065 .await
12066 .unwrap();
12067
12068 assert!(capture.events().into_iter().all(|event| {
12069 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
12070 }));
12071 }
12072
12073 struct ApproveOperator;
12074 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
12075 fn prompt(
12076 &self,
12077 _: &str,
12078 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
12079 ) -> crate::operator_confirm::ProviderResult {
12080 crate::operator_confirm::ProviderResult::Approved
12081 }
12082 }
12083
12084 #[tokio::test]
12085 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
12086 assert_eq!(
12087 ControlHandler::default().privileged_operator_prompt,
12088 PrivilegedOperatorPrompt::Log
12089 );
12090 for (initial, changed, running) in [
12091 ("log", "enforce", PrivilegedOperatorPrompt::Log),
12092 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
12093 ] {
12094 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
12095 let path = dir.join("subc.jsonc");
12096 std::fs::write(
12097 &path,
12098 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
12099 )
12100 .unwrap();
12101 let config = crate::daemon_config::load(&path).unwrap().unwrap();
12102 let registry = Arc::new(Registry::default());
12103 let handle = SupervisorHandle::new();
12104 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
12105 .with_handle(handle.clone());
12106 let handler = ControlHandler::new(registry)
12107 .with_supervisor(handle)
12108 .with_privileged_operator_prompt(config.privileged_operator_prompt)
12109 .with_supervisor_rescan(supervisor, &path, config.port);
12110 handler.forwarding.operator_confirms().configure(
12111 Arc::new(ApproveOperator),
12112 crate::operator_confirm::OperatorLimits::default(),
12113 );
12114 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
12115
12116 for (disk_mode, needs_restart) in [
12117 (Some(initial), false),
12118 (Some(changed), true),
12119 (Some(changed), true),
12120 (None, running == PrivilegedOperatorPrompt::Enforce),
12121 (Some(initial), false),
12122 ] {
12123 let mut doc = json!({"version":1});
12124 if let Some(mode) = disk_mode {
12125 doc["privileged_operator_prompt"] = json!(mode);
12126 }
12127 std::fs::write(&path, doc.to_string()).unwrap();
12128 for preview in [true, false] {
12129 let request = ClientControlRequest::SupervisorRescan { preview };
12130 let frame = Frame::build(
12131 FrameType::Request,
12132 control_flags(),
12133 0,
12134 0,
12135 1,
12136 serde_json::to_vec(&request).unwrap(),
12137 )
12138 .unwrap();
12139 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12140 assert_eq!(frames.len(), 1);
12141 let ClientControlResponse::SupervisorRescan { result } =
12142 serde_json::from_slice(&frames[0].body).unwrap()
12143 else {
12144 panic!("expected a rescan response");
12145 };
12146 let expected = if needs_restart {
12147 vec!["privileged_operator_prompt"]
12148 } else {
12149 Vec::new()
12150 };
12151 assert_eq!(result.restart_required, expected);
12152 assert_eq!(handler.privileged_operator_prompt, running);
12153 assert_eq!(
12154 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
12155 running
12156 );
12157 }
12158 }
12159 }
12160 }
12161
12162 #[tokio::test]
12163 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
12164 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
12165 let path = dir.join("subc.jsonc");
12166 let registry = Arc::new(Registry::default());
12167 let handle = SupervisorHandle::new();
12168 let supervisor =
12169 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
12170 let handler = ControlHandler::new(registry)
12171 .with_supervisor(handle)
12172 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
12173 .with_supervisor_rescan(supervisor, &path, None);
12174 handler.forwarding.operator_confirms().configure(
12175 Arc::new(ApproveOperator),
12176 crate::operator_confirm::OperatorLimits::default(),
12177 );
12178 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
12179 for preview in [true, false] {
12180 std::fs::write(
12181 &path,
12182 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
12183 )
12184 .unwrap();
12185 let request = ClientControlRequest::SupervisorRescan { preview };
12186 let frame = Frame::build(
12187 FrameType::Request,
12188 control_flags(),
12189 0,
12190 0,
12191 1,
12192 serde_json::to_vec(&request).unwrap(),
12193 )
12194 .unwrap();
12195 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12196 assert_eq!(frames.len(), 1);
12197 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
12198 assert_eq!(
12199 handler.privileged_operator_prompt,
12200 PrivilegedOperatorPrompt::Enforce
12201 );
12202 }
12203 }
12204
12205 #[tokio::test]
12210 async fn route_open_refusal_names_the_check_that_refused() {
12211 let handler = ControlHandler::new(Arc::new(Registry::default()));
12212 let capture = EventCapture::default();
12213 let _subscriber =
12214 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12215 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12216 let response = handler
12217 .handle_control_frame(
12218 &ctx,
12219 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
12220 )
12221 .await
12222 .unwrap();
12223
12224 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12225 let event = capture
12226 .events()
12227 .into_iter()
12228 .find(|event| {
12229 event.target == "control"
12230 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12231 })
12232 .expect("route.open refusal event");
12233 assert_eq!(
12234 event.fields.get("reason"),
12235 Some(&"\"not_registered\"".to_string())
12236 );
12237 }
12238
12239 #[tokio::test]
12240 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
12241 let registry = Arc::new(Registry::default());
12242 let supervisor_handle = SupervisorHandle::new();
12243 let supervisor =
12244 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12245 .with_handle(supervisor_handle.clone())
12246 .with_connection_file_path(std::env::temp_dir().join(format!(
12247 "subc-route-open-refusal-info-{}",
12248 std::process::id()
12249 )));
12250 let module = supervisor
12251 .supervise_configured(
12252 ModuleSpec {
12253 module_id: "warming".to_string(),
12254 program: fake_aft_stub_path(),
12255 args: Vec::new(),
12256 env: Vec::new(),
12257 reserved: false,
12258 reserved_prefixes: Vec::new(),
12259 protocol: ModuleProtocol::Subc,
12260 overlap: Default::default(),
12261 },
12262 true,
12263 )
12264 .unwrap();
12265 assert_eq!(module.state().unwrap(), ModuleState::Running);
12266
12267 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12268 assert!(handler
12269 .counters()
12270 .snapshot()
12271 .get("route_open_refused_by_code")
12272 .is_none());
12273 let capture = EventCapture::default();
12274 let _subscriber =
12275 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12276 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
12277 let response = handler
12278 .handle_control_frame(
12279 &ctx,
12280 route_open_frame(394, "warming", unique_project_root("refusal-info")),
12281 )
12282 .await
12283 .unwrap();
12284 module.stop().await.unwrap();
12285
12286 assert_eq!(parse_error(&response[0])["code"], "module_warming");
12287 let event = capture
12288 .events()
12289 .into_iter()
12290 .find(|event| {
12291 event.target == "control"
12292 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
12293 })
12294 .expect("route.open refusal event");
12295 assert_eq!(
12296 event.fields.get("module_id"),
12297 Some(&"\"warming\"".to_string())
12298 );
12299 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
12300 assert_eq!(
12301 event.fields.get("reason"),
12302 Some(&"\"supervised_not_registered\"".to_string())
12303 );
12304 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
12305 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
12306 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
12307 assert_eq!(
12308 handler.counters().snapshot()["route_open_refused_by_code"],
12309 json!({ "module_warming": 1 })
12310 );
12311 }
12312
12313 const OUTAGE_START: &str = "route.open refusing module: not serving";
12314 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
12315
12316 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
12317 capture
12318 .events()
12319 .into_iter()
12320 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
12321 .collect()
12322 }
12323
12324 fn supervise_stub(
12325 registry: &Arc<Registry>,
12326 module_id: &str,
12327 enabled: bool,
12328 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
12329 let supervisor_handle = SupervisorHandle::new();
12330 let supervisor =
12331 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
12332 .with_handle(supervisor_handle.clone())
12333 .with_connection_file_path(std::env::temp_dir().join(format!(
12334 "subc-route-outage-{module_id}-{}",
12335 std::process::id()
12336 )));
12337 let module = supervisor
12338 .supervise_configured(
12339 ModuleSpec {
12340 module_id: module_id.to_string(),
12341 program: fake_aft_stub_path(),
12342 args: Vec::new(),
12343 env: Vec::new(),
12344 reserved: false,
12345 reserved_prefixes: Vec::new(),
12346 protocol: ModuleProtocol::Subc,
12347 overlap: Default::default(),
12348 },
12349 enabled,
12350 )
12351 .unwrap();
12352 (supervisor_handle, module)
12353 }
12354
12355 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
12356 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
12357 module_id: module_id.to_string(),
12358 drain_timeout_ms: Some(50),
12359 })
12360 .unwrap();
12361 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
12362 }
12363
12364 #[test]
12368 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
12369 let registry = Arc::new(Registry::default());
12370 let forwarding = Arc::new(ForwardingTable::default());
12371 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12372 let second = ControlHandler::with_forwarding(registry, forwarding);
12373 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
12374 }
12375
12376 #[tokio::test(flavor = "current_thread")]
12380 async fn route_open_unknown_module_refusals_add_no_outage_state() {
12381 let handler = ControlHandler::new(Arc::new(Registry::default()));
12382 let capture = EventCapture::default();
12383 let _subscriber =
12384 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12385 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
12386 for corr in 0..8 {
12387 let response = handler
12388 .handle_control_frame(
12389 &ctx,
12390 route_open_frame(
12391 380 + corr,
12392 &format!("nobody-{corr}"),
12393 unique_project_root("outage-unknown"),
12394 ),
12395 )
12396 .await
12397 .unwrap();
12398 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12399 }
12400
12401 assert_eq!(handler.route_outages.tracked_module_count(), 0);
12402 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
12403 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
12404 }
12405
12406 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12411 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
12412 let registry = Arc::new(Registry::default());
12413 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
12414 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12415 let capture = EventCapture::default();
12416 let _subscriber =
12417 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12418 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
12419 handler.route_outages.record_accepted("outage-restart");
12422
12423 let response = handler
12424 .handle_control_frame(
12425 &ctx,
12426 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
12427 )
12428 .await
12429 .unwrap();
12430 assert_eq!(response[0].header.ty, FrameType::Error);
12431 let starts = outage_lines(&capture, OUTAGE_START);
12432 assert_eq!(starts.len(), 1, "{starts:?}");
12433 assert_eq!(starts[0].level, tracing::Level::WARN);
12434 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
12435 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
12436 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
12437 handler.route_outages.record_accepted("outage-restart");
12438 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
12439
12440 let restart = handler
12441 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
12442 .await
12443 .unwrap();
12444 assert_eq!(
12445 restart[0].header.ty,
12446 FrameType::Response,
12447 "{:?}",
12448 parse_error(&restart[0])
12449 );
12450 handler
12451 .handle_control_frame(
12452 &ctx,
12453 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
12454 )
12455 .await
12456 .unwrap();
12457 module.stop().await.unwrap();
12458
12459 let starts = outage_lines(&capture, OUTAGE_START);
12460 assert_eq!(starts.len(), 2, "{starts:?}");
12461 assert_eq!(starts[1].level, tracing::Level::INFO);
12462 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
12463 }
12464
12465 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12469 async fn failed_operator_restart_leaves_no_operator_mark() {
12470 let registry = Arc::new(Registry::default());
12471 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
12472 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12473 let capture = EventCapture::default();
12474 let _subscriber =
12475 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12476 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
12477 handler.route_outages.record_accepted("outage-disabled");
12478
12479 let restart = handler
12480 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
12481 .await
12482 .unwrap();
12483 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
12484 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
12485
12486 handler
12487 .handle_control_frame(
12488 &ctx,
12489 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
12490 )
12491 .await
12492 .unwrap();
12493 let starts = outage_lines(&capture, OUTAGE_START);
12494 assert_eq!(starts.len(), 1, "{starts:?}");
12495 assert_eq!(starts[0].level, tracing::Level::WARN);
12496 }
12497
12498 #[tokio::test(flavor = "current_thread")]
12499 async fn route_open_unknown_module_escapes_target_module_id() {
12500 let handler = ControlHandler::new(Arc::new(Registry::default()));
12501 let capture = EventCapture::default();
12502 let _subscriber =
12503 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12504 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
12505 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12506 let response = handler
12507 .handle_control_frame(
12508 &ctx,
12509 route_open_frame(
12510 395,
12511 hostile_module_id,
12512 unique_project_root("hostile-target-module-id"),
12513 ),
12514 )
12515 .await
12516 .unwrap();
12517
12518 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12519 let event = capture
12520 .events()
12521 .into_iter()
12522 .find(|event| {
12523 event.target == "control"
12524 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12525 })
12526 .expect("route.open unknown-module refusal event");
12527 let logged = event.fields.get("module_id").expect("module_id field");
12528 assert!(!logged.bytes().any(|byte| byte < 0x20));
12529 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12530 }
12531
12532 #[tokio::test(flavor = "current_thread")]
12533 async fn route_open_module_rejection_uses_daemon_counter_key() {
12534 let registry = Arc::new(Registry::default());
12535 let forwarding = Arc::new(ForwardingTable::default());
12536 let handler =
12537 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12538 let module_connection = ConnectionId::new(95);
12539 let (module_ctx, mut module_rx) = route_ctx(module_connection);
12540 hello_via_sink(
12541 &handler,
12542 &module_ctx,
12543 &mut module_rx,
12544 hello_frame("aft", PROTOCOL_VERSION, 395),
12545 )
12546 .await;
12547
12548 let client_connection = ConnectionId::new(96);
12549 let (client_ctx, _client_rx) = route_ctx(client_connection);
12550 let capture = EventCapture::default();
12551 let _subscriber =
12552 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12553 let (route_task, bind) = relay_route_open(
12554 &handler,
12555 client_connection,
12556 &client_ctx.egress,
12557 &mut module_rx,
12558 396,
12559 "aft",
12560 "hostile-module-code",
12561 )
12562 .await;
12563 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
12564 let rejection = Frame::build(
12565 FrameType::Error,
12566 control_flags(),
12567 0,
12568 0,
12569 bind.header.corr,
12570 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
12571 )
12572 .unwrap();
12573 handler
12574 .handle_control_frame(&module_ctx, rejection)
12575 .await
12576 .unwrap();
12577
12578 let response = route_task.await.unwrap();
12579 assert_eq!(parse_error(&response[0])["code"], hostile_code);
12580 let counters = handler.counters().snapshot();
12581 assert_eq!(
12582 counters["route_open_refused_by_code"],
12583 json!({ "module_rejected": 1 })
12584 );
12585 assert!(counters["route_open_refused_by_code"]
12586 .get(hostile_code)
12587 .is_none());
12588
12589 let event = capture
12590 .events()
12591 .into_iter()
12592 .find(|event| {
12593 event.target == "control"
12594 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
12595 })
12596 .expect("route.open module-rejection refusal event");
12597 let logged = event.fields.get("module_code").expect("module_code field");
12598 assert!(!logged.bytes().any(|byte| byte < 0x20));
12599 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12600 }
12601
12602 #[tokio::test]
12603 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
12604 let registry = Arc::new(Registry::default());
12605 let supervisor_handle = SupervisorHandle::new();
12606 let missing_program = std::env::temp_dir().join(format!(
12607 "subc-route-open-missing-program-{}",
12608 std::process::id()
12609 ));
12610 let supervisor =
12611 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12612 .with_handle(supervisor_handle.clone());
12613 let module = supervisor
12614 .supervise_configured(
12615 ModuleSpec {
12616 module_id: "failed".to_string(),
12617 program: missing_program,
12618 args: Vec::new(),
12619 env: Vec::new(),
12620 reserved: false,
12621 reserved_prefixes: Vec::new(),
12622 protocol: ModuleProtocol::Subc,
12623 overlap: Default::default(),
12624 },
12625 true,
12626 )
12627 .unwrap();
12628 assert_eq!(module.state().unwrap(), ModuleState::Failed);
12629
12630 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12631 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
12632 let response = handler
12633 .handle_control_frame(
12634 &ctx,
12635 route_open_frame(305, "failed", unique_project_root("failed")),
12636 )
12637 .await
12638 .unwrap();
12639
12640 assert_eq!(response[0].header.ty, FrameType::Error);
12641 let error = parse_error(&response[0]);
12642 assert_eq!(error["code"], "target_unavailable");
12643 assert!(error["message"]
12644 .as_str()
12645 .unwrap()
12646 .contains("state=failed, enabled=true, live=false"));
12647 }
12648
12649 #[tokio::test]
12650 async fn route_open_role_mismatch_remains_target_unavailable() {
12651 let registry = Arc::new(Registry::default());
12652 let handler = ControlHandler::new(Arc::clone(®istry));
12653 handler
12654 .handle_control(
12655 ConnectionId::new(41),
12656 non_routable_hello_frame_with_control_ops("health-only", 306, None),
12657 )
12658 .unwrap();
12659
12660 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
12661 let response = handler
12662 .handle_control_frame(
12663 &ctx,
12664 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
12665 )
12666 .await
12667 .unwrap();
12668
12669 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12670 assert!(parse_error(&response[0])["message"]
12671 .as_str()
12672 .unwrap()
12673 .contains("does not provide the requested target"));
12674 }
12675
12676 #[tokio::test]
12677 async fn route_open_inactive_registration_remains_target_unavailable() {
12678 let registry = Arc::new(Registry::default());
12679 let handler = ControlHandler::new(Arc::clone(®istry));
12680 handler
12681 .handle_control(
12682 ConnectionId::new(43),
12683 hello_frame("inactive", PROTOCOL_VERSION, 308),
12684 )
12685 .unwrap();
12686 assert!(registry
12687 .set_module_state_for_test("inactive", ChannelState::Closed)
12688 .unwrap());
12689
12690 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
12691 let response = handler
12692 .handle_control_frame(
12693 &ctx,
12694 route_open_frame(309, "inactive", unique_project_root("inactive")),
12695 )
12696 .await
12697 .unwrap();
12698
12699 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12700 assert!(parse_error(&response[0])["message"]
12701 .as_str()
12702 .unwrap()
12703 .contains("is not active"));
12704 }
12705
12706 #[tokio::test]
12707 async fn late_health_reply_is_recorded_through_the_module_response_path() {
12708 let registry = Arc::new(Registry::default());
12709 let forwarding = Arc::new(ForwardingTable::default());
12710 let supervisor_handle = SupervisorHandle::new();
12711 let supervisor =
12712 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
12713 .with_forwarding(Arc::clone(&forwarding))
12714 .with_handle(supervisor_handle.clone());
12715 let module = supervisor
12716 .supervise_configured(
12717 crate::ModuleSpec {
12718 module_id: "late-health-response".to_string(),
12719 program: PathBuf::from("disabled-module"),
12720 args: Vec::new(),
12721 env: Vec::new(),
12722 reserved: false,
12723 reserved_prefixes: Vec::new(),
12724 protocol: ModuleProtocol::Subc,
12725 overlap: Default::default(),
12726 },
12727 false,
12728 )
12729 .unwrap();
12730 let handler =
12731 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12732 .with_supervisor(supervisor_handle);
12733 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
12734 handler
12735 .handle_control_frame(
12736 &module_ctx,
12737 hello_frame_with_control_ops(
12738 "late-health-response",
12739 PROTOCOL_VERSION,
12740 7,
12741 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12742 ),
12743 )
12744 .await
12745 .unwrap();
12746 let probe_started_at = Instant::now() - Duration::from_millis(80);
12747 let pending = forwarding
12748 .begin_health_probe_rpc_for(
12749 "late-health-response",
12750 MODULE_CONTROL_OP_HEALTH_CHECK,
12751 probe_started_at,
12752 Instant::now() - Duration::from_millis(1),
12753 )
12754 .unwrap();
12755 assert!(forwarding
12756 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
12757 .unwrap());
12758
12759 let responses = handler
12760 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
12761 .await
12762 .unwrap();
12763
12764 assert!(responses.is_empty());
12765 let health = module.status().unwrap().health;
12766 assert_eq!(health.late_answer_count, 1);
12767 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
12768 }
12769
12770 #[tokio::test]
12771 async fn health_probe_timeout_and_module_death_are_typed() {
12772 let registry = Arc::new(Registry::default());
12773 let forwarding = Arc::new(ForwardingTable::default());
12774 let handler =
12775 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12776 .with_health_probe_timeout(Duration::from_millis(50));
12777 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12778 hello_via_sink(
12779 &handler,
12780 &module_ctx,
12781 &mut module_rx,
12782 hello_frame_with_control_ops(
12783 "aft",
12784 PROTOCOL_VERSION,
12785 7,
12786 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12787 ),
12788 )
12789 .await;
12790
12791 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12792 let responses = handler
12793 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
12794 .await
12795 .unwrap();
12796 assert_eq!(responses[0].header.ty, FrameType::Error);
12797 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
12798 let _ = module_rx.try_recv();
12799
12800 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
12801 let health_handler = handler.clone();
12802 let death_task = tokio::spawn(async move {
12803 health_handler
12804 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
12805 .await
12806 .unwrap()
12807 });
12808 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12809 .await
12810 .unwrap()
12811 .unwrap();
12812 handler
12813 .cleanup_connection(module_ctx.connection_id)
12814 .unwrap();
12815 let responses = death_task.await.unwrap();
12816 assert_eq!(responses[0].header.ty, FrameType::Error);
12817 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
12818 }
12819
12820 #[test]
12821 fn hello_requires_exact_protocol_version() {
12822 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
12823 let registry = Arc::new(Registry::default());
12824 let handler = ControlHandler::new(Arc::clone(®istry));
12825 let responses = handler
12826 .handle_control(
12827 ConnectionId::new(connection),
12828 hello_frame("aft", offered, 9),
12829 )
12830 .unwrap();
12831
12832 assert_eq!(responses.len(), 1);
12833 assert_eq!(responses[0].header.ty, FrameType::Error);
12834 let error = parse_error(&responses[0]);
12835 assert_eq!(error["code"], "version_unsupported");
12836 assert!(registry.get_module("aft").unwrap().is_none());
12837 assert_eq!(registry.active_registration_count().unwrap(), 0);
12838 }
12839 }
12840
12841 #[test]
12842 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
12843 let registry = Arc::new(Registry::default());
12844 let forwarding = Arc::new(ForwardingTable::default());
12845 let handler =
12846 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12847 let module_connection = ConnectionId::new(301);
12848 let registration = registry
12849 .register_with_control_ops(
12850 manifest("aft-push", PROTOCOL_VERSION),
12851 PROTOCOL_VERSION,
12852 module_connection,
12853 module_baseline_control_ops(),
12854 )
12855 .unwrap();
12856 let (module_tx, _module_rx) = mpsc::channel(8);
12857 let endpoint = forwarding
12858 .register_module_connection(
12859 module_connection,
12860 "aft-push".to_string(),
12861 PROTOCOL_VERSION,
12862 manifest_concurrency(®istration.manifest),
12863 FrameSink::new(module_tx),
12864 )
12865 .unwrap();
12866
12867 let unknown = Frame::build(
12869 FrameType::Push,
12870 control_flags(),
12871 0,
12872 0,
12873 5,
12874 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
12875 )
12876 .unwrap();
12877 let out = handler.handle_status_update(endpoint, unknown).unwrap();
12878 assert!(
12879 out.is_empty(),
12880 "unknown push op must be ignored, got {out:?}"
12881 );
12882
12883 let malformed = Frame::build(
12885 FrameType::Push,
12886 control_flags(),
12887 0,
12888 0,
12889 6,
12890 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
12891 )
12892 .unwrap();
12893 let out = handler.handle_status_update(endpoint, malformed).unwrap();
12894 assert_eq!(out.len(), 1);
12895 assert_eq!(out[0].header.ty, FrameType::Error);
12896 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
12897 }
12898
12899 #[test]
12900 fn hello_rejected_when_connection_already_owns_client_routes() {
12901 let registry = Arc::new(Registry::default());
12902 let forwarding = Arc::new(ForwardingTable::default());
12903 let handler =
12904 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12905 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
12907 let client_connection = ConnectionId::new(202);
12908
12909 let responses = handler
12912 .handle_control(
12913 client_connection,
12914 hello_frame("aft-second", PROTOCOL_VERSION, 9),
12915 )
12916 .unwrap();
12917 assert_eq!(responses[0].header.ty, FrameType::Error);
12918 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
12919 assert!(registry.get_module("aft-second").unwrap().is_none());
12920 }
12921
12922 #[tokio::test]
12923 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
12924 let registry = Arc::new(Registry::default());
12925 let forwarding = Arc::new(ForwardingTable::default());
12926 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
12927 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
12928 hello_via_sink(
12929 &handler,
12930 &module_ctx,
12931 &mut module_rx,
12932 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
12933 )
12934 .await;
12935 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
12936 let pending = forwarding
12937 .begin_route_bind_relay_for_test(
12938 client_ctx.connection_id,
12939 client_ctx.egress.clone(),
12940 2,
12941 "alpha",
12942 )
12943 .unwrap();
12944 forwarding
12945 .complete_pending_relay(
12946 module_ctx.connection_id,
12947 pending.corr,
12948 RouteBindRelayOutcome::Accepted,
12949 )
12950 .unwrap();
12951 client_rx.try_recv().unwrap();
12952 for module_id in ["beta", "alpha"] {
12953 let replies = handler
12954 .handle_control_frame(
12955 &module_ctx,
12956 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
12957 )
12958 .await
12959 .unwrap();
12960 assert_eq!(replies.len(), 1, "second HELLO must be refused");
12961 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
12962 }
12963 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
12964 assert!(registry.get_module("beta").unwrap().is_none());
12965 assert!(matches!(
12966 forwarding
12967 .lookup_data_route(
12968 client_ctx.connection_id,
12969 pending.client_channel,
12970 pending.client_epoch,
12971 )
12972 .unwrap(),
12973 DataRoute::Client(DataRouteState::Bound(_))
12974 ));
12975 assert!(handler
12976 .hello_launch_nonces
12977 .lock()
12978 .unwrap()
12979 .presented(module_ctx.connection_id, Some("alpha-nonce")));
12980 assert!(module_rx.try_recv().is_err());
12981 }
12982
12983 #[test]
12984 fn reserved_module_hello_requires_matching_launch_nonce() {
12985 let registry = Arc::new(Registry::default());
12986 let supervisor = SupervisorHandle::new();
12987 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12990 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12991
12992 let no_nonce = handler
12994 .handle_control(
12995 ConnectionId::new(1),
12996 hello_frame("vault", PROTOCOL_VERSION, 1),
12997 )
12998 .unwrap();
12999 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
13000 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
13001 assert!(registry.get_module("vault").unwrap().is_none());
13002
13003 let wrong = handler
13005 .handle_control(
13006 ConnectionId::new(2),
13007 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
13008 )
13009 .unwrap();
13010 assert_eq!(wrong[0].header.ty, FrameType::Error);
13011 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
13012 assert!(registry.get_module("vault").unwrap().is_none());
13013
13014 let ok = handler
13016 .handle_control(
13017 ConnectionId::new(3),
13018 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
13019 )
13020 .unwrap();
13021 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
13022 assert!(registry.get_module("vault").unwrap().is_some());
13023 }
13024
13025 #[test]
13026 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
13027 let registry = Arc::new(Registry::default());
13028 let supervisor = SupervisorHandle::new();
13029 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13030 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13031 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13032
13033 let squat = handler
13034 .handle_control(
13035 ConnectionId::new(1),
13036 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
13037 )
13038 .unwrap();
13039 assert_eq!(squat[0].header.ty, FrameType::Error);
13040 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
13041 assert!(parse_error(&squat[0])["message"]
13042 .as_str()
13043 .unwrap()
13044 .contains("fed:"));
13045
13046 let accepted_peer = handler
13047 .handle_control(
13048 ConnectionId::new(2),
13049 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
13050 )
13051 .unwrap();
13052 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
13053
13054 let accepted_short = handler
13055 .handle_control(
13056 ConnectionId::new(3),
13057 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
13058 )
13059 .unwrap();
13060 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
13061
13062 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
13063 let response = handler
13064 .handle_control(
13065 ConnectionId::new(conn),
13066 hello_frame(module_id, PROTOCOL_VERSION, conn),
13067 )
13068 .unwrap();
13069 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
13070 }
13071 }
13072
13073 #[test]
13074 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
13075 let registry = Arc::new(Registry::default());
13076 let supervisor = SupervisorHandle::new();
13077 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
13078 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
13079 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
13080 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
13081
13082 let owner_nonce = handler
13083 .handle_control(
13084 ConnectionId::new(1),
13085 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
13086 )
13087 .unwrap();
13088 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
13089 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
13090 assert!(registry.get_module("fed:special").unwrap().is_none());
13091
13092 let exact_nonce = handler
13093 .handle_control(
13094 ConnectionId::new(2),
13095 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
13096 )
13097 .unwrap();
13098 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
13099 assert!(registry.get_module("fed:special").unwrap().is_some());
13100 }
13101
13102 #[test]
13103 fn non_reserved_module_ignores_launch_nonce() {
13104 let registry = Arc::new(Registry::default());
13105 let handler = ControlHandler::new(Arc::clone(®istry));
13108 let no_nonce = handler
13109 .handle_control(
13110 ConnectionId::new(1),
13111 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
13112 )
13113 .unwrap();
13114 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
13115 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
13116
13117 let echoed_nonce = handler
13118 .handle_control(
13119 ConnectionId::new(2),
13120 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
13121 )
13122 .unwrap();
13123 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
13124 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
13125 }
13126
13127 #[test]
13128 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
13129 let handler = ControlHandler::default();
13130 let conn = ConnectionId::new(1);
13131 let malformed = Frame::build(
13132 FrameType::Hello,
13133 control_flags(),
13134 0,
13135 0,
13136 3,
13137 b"{not json".to_vec(),
13138 )
13139 .unwrap();
13140
13141 let error = handler.handle_control(conn, malformed).unwrap();
13142 assert_eq!(error[0].header.ty, FrameType::Error);
13143 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
13144
13145 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13146 let pong = handler.handle_control(conn, ping).unwrap();
13147 assert_eq!(pong[0].header.ty, FrameType::Pong);
13148 assert_eq!(pong[0].header.corr, 4);
13149 }
13150
13151 #[test]
13152 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
13153 let registry = Arc::new(Registry::default());
13154 let handler = ControlHandler::new(Arc::clone(®istry));
13155
13156 handler
13157 .handle_control(
13158 ConnectionId::new(1),
13159 hello_frame("aft", PROTOCOL_VERSION, 1),
13160 )
13161 .unwrap();
13162 let duplicate = handler
13163 .handle_control(
13164 ConnectionId::new(2),
13165 hello_frame("aft", PROTOCOL_VERSION, 2),
13166 )
13167 .unwrap();
13168
13169 assert_eq!(duplicate[0].header.ty, FrameType::Error);
13170 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13171 let registration = registry.get_module("aft").unwrap().unwrap();
13172 assert_eq!(registration.connection_id, ConnectionId::new(1));
13173 }
13174
13175 #[test]
13176 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
13177 let registry = Arc::new(Registry::default());
13178 let forwarding = Arc::new(ForwardingTable::default());
13179 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
13180 let handler =
13181 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13182 .with_process_liveness(process_liveness);
13183 let (ctx, route_channel, route_epoch) =
13184 bind_liveness_route(®istry, &forwarding, "aft-dead");
13185 let responses = handler
13186 .handle_route_poll(
13187 &ctx,
13188 route_poll_frame(41, PollKind::Liveness, route_channel),
13189 route_channel,
13190 route_epoch,
13191 PollKind::Liveness,
13192 )
13193 .unwrap();
13194
13195 assert_eq!(responses.len(), 1);
13196 assert_eq!(responses[0].header.ty, FrameType::Response);
13197 assert_route_poll_liveness(&responses[0], false);
13198 }
13199
13200 #[test]
13201 fn liveness_poll_without_process_source_uses_bound_route() {
13202 let registry = Arc::new(Registry::default());
13203 let forwarding = Arc::new(ForwardingTable::default());
13204 let handler =
13205 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13206 let (ctx, route_channel, route_epoch) =
13207 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
13208 let responses = handler
13209 .handle_route_poll(
13210 &ctx,
13211 route_poll_frame(42, PollKind::Liveness, route_channel),
13212 route_channel,
13213 route_epoch,
13214 PollKind::Liveness,
13215 )
13216 .unwrap();
13217
13218 assert_route_poll_liveness(&responses[0], true);
13219 }
13220
13221 #[test]
13222 fn liveness_poll_untracked_process_source_uses_bound_route() {
13223 let registry = Arc::new(Registry::default());
13224 let forwarding = Arc::new(ForwardingTable::default());
13225 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
13226 let handler =
13227 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13228 .with_process_liveness(process_liveness);
13229 let (ctx, route_channel, route_epoch) =
13230 bind_liveness_route(®istry, &forwarding, "aft-untracked");
13231 let responses = handler
13232 .handle_route_poll(
13233 &ctx,
13234 route_poll_frame(43, PollKind::Liveness, route_channel),
13235 route_channel,
13236 route_epoch,
13237 PollKind::Liveness,
13238 )
13239 .unwrap();
13240
13241 assert_route_poll_liveness(&responses[0], true);
13242 }
13243
13244 #[tokio::test]
13245 async fn unknown_op_returns_unknown_control_op() {
13246 let handler = ControlHandler::default();
13247 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
13248 let request = Frame::build(
13249 FrameType::Request,
13250 control_flags(),
13251 0,
13252 0,
13253 55,
13254 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
13255 )
13256 .unwrap();
13257
13258 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13259
13260 assert_eq!(response.len(), 1);
13261 assert_eq!(response[0].header.ty, FrameType::Error);
13262 assert_eq!(response[0].header.corr, 55);
13263 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
13264 }
13265
13266 #[tokio::test]
13267 async fn supervisor_provenance_rejects_unknown_exact_module() {
13268 let handler = ControlHandler::default();
13269 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
13270 let request = Frame::build(
13271 FrameType::Request,
13272 control_flags(),
13273 0,
13274 0,
13275 57,
13276 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
13277 )
13278 .unwrap();
13279
13280 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13281
13282 assert_eq!(response.len(), 1);
13283 assert_eq!(response[0].header.ty, FrameType::Error);
13284 assert_eq!(response[0].header.corr, 57);
13285 let error = parse_error(&response[0]);
13286 assert_eq!(error["code"], "unknown_module");
13287 assert_eq!(error["message"], "module_id 'missing' is not supervised");
13288 }
13289
13290 #[test]
13291 fn provenance_probe_override_keeps_handler_tests_deterministic() {
13292 let expected = subc_control::RunningImageAgreement::Unavailable {
13293 reason: subc_control::RunningImageUnavailableReason::HashFailed,
13294 };
13295 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
13296 assert_eq!(handler.provenance_probe_override, Some(expected));
13297 }
13298
13299 #[test]
13300 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
13301 let verdict = reload_verdict(
13302 std::path::Path::new("/bin/new"),
13303 Some(std::path::Path::new("/bin/old")),
13304 subc_control::RunningImageAgreement::Unavailable {
13305 reason: subc_control::RunningImageUnavailableReason::HashFailed,
13306 },
13307 );
13308 assert!(matches!(
13309 verdict.path,
13310 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
13311 if configured == std::path::Path::new("/bin/new")
13312 && spawned_from == std::path::Path::new("/bin/old")
13313 ));
13314 }
13315
13316 #[test]
13317 fn launch_spec_without_running_digest_is_unavailable_not_match() {
13318 use subc_control::{
13319 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
13320 };
13321 let configured = crate::supervise::ModuleSpec {
13322 module_id: "legacy".into(),
13323 program: "/module".into(),
13324 args: Vec::new(),
13325 env: Vec::new(),
13326 reserved: false,
13327 reserved_prefixes: Vec::new(),
13328 protocol: subc_control::ModuleProtocol::Subc,
13329 overlap: Default::default(),
13330 };
13331 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
13332 assert_eq!(
13333 launch_spec_agreement(&configured, false, Some(&digest)),
13334 Agreement::Unavailable {
13335 reason: Reason::NotRunning
13336 }
13337 );
13338 assert_eq!(
13339 launch_spec_agreement(&configured, true, None),
13340 Agreement::Unavailable {
13341 reason: Reason::SpawnedSpecUnavailable
13342 }
13343 );
13344 assert_eq!(
13345 launch_spec_agreement(&configured, true, Some(&digest)),
13346 Agreement::Match
13347 );
13348 }
13349
13350 #[cfg(windows)]
13351 #[test]
13352 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
13353 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
13354 let configured = dir.join("module.exe");
13355 std::fs::write(&configured, b"executable path fixture").unwrap();
13356 let spawned = std::fs::canonicalize(&configured).unwrap();
13357 assert_ne!(
13358 configured, spawned,
13359 "fixture must exercise different path spellings"
13360 );
13361 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
13362 let lower_case = configured.to_string_lossy().to_lowercase();
13363 for spelling in [
13364 &configured,
13365 Path::new(&forward_slashes),
13366 Path::new(&lower_case),
13367 ] {
13368 let verdict = reload_verdict(
13369 spelling,
13370 Some(&spawned),
13371 subc_control::RunningImageAgreement::Unavailable {
13372 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13373 },
13374 );
13375 assert_eq!(
13376 verdict.path,
13377 ReloadPathAgreement::Match,
13378 "configured {spelling:?}, spawned {spawned:?}"
13379 );
13380 }
13381 }
13382
13383 #[cfg(windows)]
13384 #[test]
13385 fn windows_reload_path_keeps_different_existing_program_pending() {
13386 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
13387 let old = dir.join("old.exe");
13388 let new = dir.join("new.exe");
13389 std::fs::write(&old, b"same executable content").unwrap();
13390 std::fs::copy(&old, &new).unwrap();
13391 let spawned = std::fs::canonicalize(&old).unwrap();
13392 let image = subc_control::RunningImageAgreement::Unavailable {
13393 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13394 };
13395 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
13396 assert!(
13397 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
13398 if configured == new && spawned_from == spawned)
13399 );
13400 let missing = dir.join("missing.exe");
13401 assert!(matches!(
13402 reload_verdict(&missing, Some(&spawned), image).path,
13403 ReloadPathAgreement::Mismatch { .. }
13404 ));
13405 }
13406
13407 #[test]
13408 fn reload_verdict_detects_replaced_image_at_same_path() {
13409 let image = subc_control::RunningImageAgreement::Mismatch {
13410 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
13411 digest: "old".into(),
13412 },
13413 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
13414 digest: "new".into(),
13415 },
13416 };
13417 let verdict = reload_verdict(
13418 std::path::Path::new("/bin/same"),
13419 Some(std::path::Path::new("/bin/same")),
13420 image.clone(),
13421 );
13422 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13423 assert_eq!(verdict.image, image);
13424 }
13425
13426 #[test]
13427 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
13428 let image = subc_control::RunningImageAgreement::Unavailable {
13429 reason: subc_control::RunningImageUnavailableReason::NotRunning,
13430 };
13431 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
13432 assert_eq!(
13433 verdict.path,
13434 subc_control::ReloadPathAgreement::Unavailable {
13435 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
13436 }
13437 );
13438 assert_eq!(verdict.image, image);
13439
13440 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
13441 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13442 };
13443 let verdict = reload_verdict(
13444 std::path::Path::new("/bin/same"),
13445 Some(std::path::Path::new("/bin/same")),
13446 unconfirmed.clone(),
13447 );
13448 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13449 assert_eq!(verdict.image, unconfirmed);
13450 }
13451
13452 #[test]
13453 fn reload_verdict_preserves_each_image_unavailability_reason() {
13454 use subc_control::RunningImageUnavailableReason as Reason;
13455
13456 for reason in [
13457 Reason::NotRunning,
13458 Reason::UnsupportedPlatform,
13459 Reason::RunningExecutableUnreadable,
13460 Reason::SpawnedPathUnreadable,
13461 Reason::HashFailed,
13462 Reason::ProcessIdentityUnconfirmed,
13463 Reason::Unknown("future_probe_reason".to_string()),
13464 ] {
13465 let image = subc_control::RunningImageAgreement::Unavailable {
13466 reason: reason.clone(),
13467 };
13468 let verdict = reload_verdict(
13469 std::path::Path::new("/bin/same"),
13470 Some(std::path::Path::new("/bin/same")),
13471 image.clone(),
13472 );
13473 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13474 assert_eq!(verdict.image, image, "{reason:?}");
13475 }
13476 }
13477
13478 #[tokio::test]
13479 async fn malformed_control_bodies_return_invalid_control_body() {
13480 let handler = ControlHandler::default();
13481 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
13482
13483 for (corr, body) in [
13484 (56, br#"{"route_channel":1}"#.as_slice()),
13485 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
13486 (
13487 58,
13488 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
13489 ),
13490 ] {
13491 let request = Frame::build(
13492 FrameType::Request,
13493 control_flags(),
13494 0,
13495 0,
13496 corr,
13497 body.to_vec(),
13498 )
13499 .unwrap();
13500 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13501
13502 assert_eq!(response.len(), 1);
13503 assert_eq!(response[0].header.ty, FrameType::Error);
13504 assert_eq!(response[0].header.corr, corr);
13505 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
13506 }
13507 }
13508
13509 #[tokio::test]
13510 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
13511 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13512 let registry = Arc::new(Registry::default());
13513 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13514 let router = Router::with_control_handler(Arc::clone(&control));
13515 let connection = router.begin_connection();
13516 let (ctx, mut rx) = route_ctx(connection.id());
13517
13518 router
13519 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
13520 .await
13521 .unwrap();
13522 let response = rx.recv().await.unwrap();
13523 let ack = parse_ack(&response);
13524 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13525 let channel = 1;
13526
13527 let goodbye =
13528 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
13529 router.route_for_connection(&ctx, goodbye).await.unwrap();
13530 assert!(rx.try_recv().is_err());
13531 assert!(registry.get_module("aft").unwrap().is_none());
13532
13533 router
13534 .route_for_connection(&ctx, channel_request(channel, 13))
13535 .await
13536 .unwrap();
13537 let error_frame = rx.recv().await.unwrap();
13538 assert_eq!(error_frame.header.ty, FrameType::Error);
13539 assert_eq!(error_frame.header.channel, channel);
13540 let captured = crate::router::test_log::captured_logs(&logs);
13541 assert_eq!(
13542 captured
13543 .lines()
13544 .filter(|line| {
13545 line.contains("module_id=aft")
13546 && line.contains("reason=explicit_goodbye")
13547 && line.contains("module registration ended")
13548 })
13549 .count(),
13550 1,
13551 "unexpected GOODBYE registry log: {captured}"
13552 );
13553 }
13554
13555 #[tokio::test]
13556 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
13557 let registry = Arc::new(Registry::default());
13558 let handler = ControlHandler::new(registry).with_capability_config(
13559 [("prov".to_string(), true), ("cons".to_string(), true)],
13560 BTreeMap::new(),
13561 );
13562 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
13563 register_capability_manifest(
13564 &handler,
13565 &provider_ctx,
13566 &mut provider_rx,
13567 capability_manifest("prov", &["thing/v1"], &[]),
13568 1,
13569 )
13570 .await;
13571 let mut consumer = capability_manifest("cons", &[], &[]);
13572 consumer.capabilities.as_mut().unwrap().requires.push(
13573 subc_protocol::manifest::CapabilityRequirement {
13574 capability: "thing/v1".to_string(),
13575 need: subc_protocol::manifest::CapabilityNeed::Required,
13576 },
13577 );
13578 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
13579 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
13580 assert_eq!(
13581 handler.capability_evaluator.verdict("cons", "thing/v1"),
13582 Some(CapabilityVerdict::Provided)
13583 );
13584 let (mut client_rx, _) = open_route_for_capability_test(
13585 &handler,
13586 &provider_ctx,
13587 &mut provider_rx,
13588 703,
13589 3,
13590 "prov",
13591 None,
13592 )
13593 .await;
13594 let goodbye =
13595 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13596 handler
13597 .handle_control_frame(&provider_ctx, goodbye)
13598 .await
13599 .unwrap();
13600 assert_eq!(
13601 handler.capability_evaluator.verdict("cons", "thing/v1"),
13602 Some(CapabilityVerdict::NeverProvided)
13603 );
13604 let closed = client_rx
13605 .try_recv()
13606 .expect("GOODBYE pushes route.closed before route GOODBYE");
13607 assert!(
13608 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
13609 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
13610 );
13611 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
13612 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
13613 }
13614
13615 struct RouteClosedFixture {
13616 handler: ControlHandler,
13617 #[cfg(unix)]
13618 supervisor: Supervisor,
13619 module_ctx: RouteCtx,
13620 #[cfg(unix)]
13621 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13622 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13623 route: Arc<crate::forwarding::RouteBinding>,
13624 }
13625
13626 async fn route_closed_fixture() -> RouteClosedFixture {
13627 let registry = Arc::new(Registry::default());
13628 let handler = ControlHandler::new(Arc::clone(®istry));
13629 #[cfg(unix)]
13630 let supervisor = Supervisor::new(registry, RestartPolicy::default())
13631 .with_forwarding(Arc::clone(&handler.forwarding));
13632 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
13633 register_capability_manifest(
13634 &handler,
13635 &module_ctx,
13636 &mut module_rx,
13637 capability_manifest("prov", &[], &[]),
13638 1,
13639 )
13640 .await;
13641 let (client_rx, bind) = open_route_for_capability_test(
13642 &handler,
13643 &module_ctx,
13644 &mut module_rx,
13645 702,
13646 2,
13647 "prov",
13648 None,
13649 )
13650 .await;
13651 let ModuleControlRequest::RouteBind {
13652 route_channel: channel,
13653 epoch: route_epoch,
13654 ..
13655 } = bind
13656 else {
13657 panic!("expected route.bind");
13658 };
13659 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
13660 handler
13661 .forwarding
13662 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
13663 .unwrap()
13664 else {
13665 panic!("the acknowledged route must be bound");
13666 };
13667 RouteClosedFixture {
13668 handler,
13669 #[cfg(unix)]
13670 supervisor,
13671 module_ctx,
13672 #[cfg(unix)]
13673 module_rx,
13674 client_rx,
13675 route,
13676 }
13677 }
13678
13679 impl RouteClosedFixture {
13680 #[cfg(unix)]
13681 async fn cut(&mut self) {
13682 let notices = async {
13683 let mut draining = self.module_rx.recv().await.unwrap();
13684 assert!(matches!(
13685 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
13686 &draining.body
13687 )
13688 .unwrap(),
13689 subc_protocol::session::ModuleControlCommand::Draining {
13690 reason: RouteCloseReason::Restart,
13691 ..
13692 }
13693 ));
13694 draining.flushed.take().unwrap().send(()).unwrap();
13695 let mut closing = self.client_rx.recv().await.unwrap();
13696 assert!(matches!(
13697 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
13698 ClientControlPush::RouteClosing {
13699 reason: RouteCloseReason::Restart,
13700 ..
13701 }
13702 ));
13703 closing.flushed.take().unwrap().send(()).unwrap();
13704 };
13705 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
13706 result.unwrap();
13707 }
13708
13709 fn disconnect(&mut self) -> ClientControlPush {
13710 self.handler
13711 .cleanup_connection(self.module_ctx.connection_id)
13712 .unwrap();
13713 let closed = self.client_rx.try_recv().unwrap();
13714 assert_eq!(closed.header.ty, FrameType::Push);
13715 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
13716 assert!(matches!(
13717 &push,
13718 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
13719 if channels == &[self.route.client_channel]
13720 ));
13721 assert_eq!(
13722 self.client_rx.try_recv().unwrap().header.ty,
13723 FrameType::Goodbye
13724 );
13725 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
13726 push
13727 }
13728 }
13729
13730 #[cfg(unix)]
13731 #[tokio::test(start_paused = true)]
13732 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
13733 let mut fixture = route_closed_fixture().await;
13734 fixture.cut().await;
13735 assert!(matches!(
13736 fixture.disconnect(),
13737 ClientControlPush::RouteClosed {
13738 reason: RouteCloseReason::Restart,
13739 drained: true,
13740 ..
13741 }
13742 ));
13743 }
13744
13745 #[cfg(unix)]
13746 #[tokio::test(start_paused = true)]
13747 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
13748 let mut fixture = route_closed_fixture().await;
13749 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
13750 let started = tokio::time::Instant::now();
13751 fixture.cut().await;
13752 assert!(started.elapsed() >= Duration::from_secs(2));
13753 assert_eq!(fixture.route.flow.in_flight(), 1);
13754 assert!(matches!(
13755 fixture.disconnect(),
13756 ClientControlPush::RouteClosed {
13757 reason: RouteCloseReason::Restart,
13758 drained: false,
13759 ..
13760 }
13761 ));
13762 }
13763
13764 #[tokio::test]
13765 async fn module_connection_crash_reports_crash_and_not_drained() {
13766 let mut fixture = route_closed_fixture().await;
13767 assert!(matches!(
13771 fixture.disconnect(),
13772 ClientControlPush::RouteClosed {
13773 reason: RouteCloseReason::Crash,
13774 drained: false,
13775 ..
13776 }
13777 ));
13778 }
13779
13780 #[tokio::test]
13781 async fn dropping_router_connection_releases_registration() {
13782 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13783 let registry = Arc::new(Registry::default());
13784 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13785 let router = Router::with_control_handler(Arc::clone(&control));
13786 let connection = router.begin_connection();
13787 let connection_id = connection.id();
13788 let (ctx, mut rx) = route_ctx(connection_id);
13789
13790 router
13791 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
13792 .await
13793 .unwrap();
13794 let response = rx.recv().await.unwrap();
13795 let ack = parse_ack(&response);
13796 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13797 assert!(registry.get_module("aft").unwrap().is_some());
13798
13799 drop(connection);
13800
13801 assert!(registry.get_module("aft").unwrap().is_none());
13802 assert_eq!(registry.active_registration_count().unwrap(), 0);
13803
13804 control
13805 .cleanup_connection(ConnectionId::new(u64::MAX))
13806 .unwrap();
13807 let captured = crate::router::test_log::captured_logs(&logs);
13808 println!("captured registry lifecycle logs:\n{captured}");
13809 let events: Vec<_> = captured
13810 .lines()
13811 .filter(|line| {
13812 line.contains("module registered module_id=aft ")
13813 || line.contains("module registration ended")
13814 })
13815 .collect();
13816 assert_eq!(
13817 events.len(),
13818 2,
13819 "unexpected registry lifecycle logs: {captured}"
13820 );
13821 assert!(events[0].contains("module registered module_id=aft "));
13822 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
13823 assert!(events[1].contains(&format!(
13824 "module_id=aft connection_id={}",
13825 connection_id.get()
13826 )));
13827 assert!(events[1].contains("reason=connection_closed"));
13828 assert!(events[1].contains("module registration ended"));
13829 assert_eq!(
13830 captured
13831 .lines()
13832 .filter(|line| line.contains("module registered module_id=aft "))
13833 .count(),
13834 1,
13835 "legacy registration admission line must appear once: {captured}"
13836 );
13837 }
13838
13839 #[tokio::test]
13840 async fn hello_registration_keeps_legacy_module_registered_line_once() {
13841 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13842 let registry = Arc::new(Registry::default());
13843 let control = ControlHandler::new(Arc::clone(®istry));
13844 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
13845 hello_via_sink(
13846 &control,
13847 &ctx,
13848 &mut rx,
13849 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
13850 )
13851 .await;
13852
13853 let captured = crate::router::test_log::captured_logs(&logs);
13854 let admissions: Vec<_> = captured
13855 .lines()
13856 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
13857 .collect();
13858 assert_eq!(
13859 admissions.len(),
13860 1,
13861 "legacy admission line must remain exactly once: {captured}"
13862 );
13863 assert!(admissions[0].contains("routable_provider=true"));
13864 assert!(admissions[0].contains("connection_id=777"));
13865 }
13866
13867 fn capability_manifest(
13868 module_id: &str,
13869 provides: &[&str],
13870 must_never_reach: &[&str],
13871 ) -> ModuleManifest {
13872 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
13873 manifest.capabilities = Some(CapabilityDeclarations {
13874 provides: provides
13875 .iter()
13876 .map(|capability| (*capability).to_string())
13877 .collect(),
13878 requires: Vec::new(),
13879 must_never_reach: must_never_reach
13880 .iter()
13881 .map(|capability| (*capability).to_string())
13882 .collect(),
13883 });
13884 manifest
13885 }
13886
13887 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
13888 Frame::build(
13889 FrameType::Hello,
13890 control_flags(),
13891 0,
13892 0,
13893 corr,
13894 serde_json::to_vec(&ModuleHelloBody {
13895 protocol_ver: manifest.protocol_ver,
13896 manifest,
13897 control_ops: None,
13898 launch_nonce: None,
13899 })
13900 .expect("capability test HELLO serializes"),
13901 )
13902 .expect("capability test HELLO frame builds")
13903 }
13904
13905 fn catalog_update_with_capabilities_frame(
13906 corr: u64,
13907 capabilities: CapabilityDeclarations,
13908 ) -> Frame {
13909 Frame::build(
13910 FrameType::Request,
13911 control_flags(),
13912 0,
13913 0,
13914 corr,
13915 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13916 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
13917 capabilities: Some(capabilities),
13918 ready: None,
13919 })
13920 .expect("capability catalog.update serializes"),
13921 )
13922 .expect("capability catalog.update frame builds")
13923 }
13924
13925 async fn register_capability_manifest(
13926 handler: &ControlHandler,
13927 ctx: &RouteCtx,
13928 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13929 manifest: ModuleManifest,
13930 corr: u64,
13931 ) {
13932 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
13933 }
13934
13935 async fn open_route_for_capability_test(
13936 handler: &ControlHandler,
13937 target_ctx: &RouteCtx,
13938 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13939 client_connection_id: u64,
13940 corr: u64,
13941 target_module_id: &str,
13942 consumer_identity: Option<ConsumerIdentity>,
13943 ) -> (
13944 mpsc::Receiver<crate::router::OutboundFrame>,
13945 ModuleControlRequest,
13946 ) {
13947 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
13948 let route_handler = handler.clone();
13949 let target_module_id = target_module_id.to_string();
13950 let route_task = tokio::spawn(async move {
13951 route_handler
13952 .handle_control_frame(
13953 &client_ctx,
13954 route_open_frame_with_admission_facts(
13955 corr,
13956 &target_module_id,
13957 unique_project_root("admission-facts"),
13958 consumer_identity,
13959 None,
13960 ),
13961 )
13962 .await
13963 .expect("capability test route.open succeeds")
13964 });
13965 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
13966 .await
13967 .expect("capability test route.open must reach route.bind")
13968 .expect("target control receiver stays open");
13969 let bind_request: ModuleControlRequest =
13970 serde_json::from_slice(&bind.body).expect("route.bind decodes");
13971 handler
13972 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
13973 .await
13974 .expect("capability test route.bind ACK succeeds");
13975 assert!(route_task.await.expect("route.open task joins").is_empty());
13976 let opened = client_rx
13977 .recv()
13978 .await
13979 .expect("successful route.open publishes a response");
13980 assert!(matches!(
13981 serde_json::from_slice::<ClientControlResponse>(&opened.body),
13982 Ok(ClientControlResponse::RouteOpen { .. })
13983 ));
13984 (client_rx, bind_request)
13985 }
13986
13987 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
13988 assert_eq!(frame.header.ty, FrameType::Push);
13989 assert_eq!(frame.header.channel, 0);
13990 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
13991 .expect("route.closed control push decodes");
13992 let ClientControlPush::RouteClosed { channels, .. } = &push else {
13993 panic!("expected route.closed");
13994 };
13995 assert_eq!(channels.len(), 1, "exactly one violating route closed");
13996 let channels = channels.clone();
13997 assert_eq!(
13998 push,
13999 ClientControlPush::RouteClosed {
14000 module_id: target_module_id.to_string(),
14001 channels,
14002 reason: RouteCloseReason::CapabilityDenied,
14003 drained: false,
14004 abandoned: 0,
14005 excluded_subscriptions: 0,
14006 terminal: Some(false),
14007 }
14008 );
14009 }
14010
14011 #[tokio::test]
14012 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
14013 let registry = Arc::new(Registry::default());
14014 let forwarding = Arc::new(ForwardingTable::default());
14015 let supervisor = SupervisorHandle::new();
14016 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14017 let handler =
14018 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14019 .with_supervisor(supervisor);
14020 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
14021 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
14022 register_capability_manifest(
14023 &handler,
14024 &target_ctx,
14025 &mut target_rx,
14026 capability_manifest("target", &["credentials-provider/v1"], &[]),
14027 1,
14028 )
14029 .await;
14030 register_capability_manifest(
14031 &handler,
14032 &opener_ctx,
14033 &mut opener_rx,
14034 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14035 2,
14036 )
14037 .await;
14038
14039 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
14040 let replies = handler
14041 .handle_control_frame(
14042 &client_ctx,
14043 route_open_frame_with_admission_facts(
14044 3,
14045 "target",
14046 unique_project_root("admission-facts"),
14047 Some(ConsumerIdentity {
14048 module_id: "opener".to_string(),
14049 launch_nonce: "opener-nonce".to_string(),
14050 }),
14051 None,
14052 ),
14053 )
14054 .await
14055 .expect("denied route.open returns a typed frame");
14056 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14057 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14058 assert!(
14059 target_rx.try_recv().is_err(),
14060 "forbidden route.open must not relay route.bind"
14061 );
14062 }
14063
14064 #[tokio::test]
14065 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
14066 let registry = Arc::new(Registry::default());
14067 let forwarding = Arc::new(ForwardingTable::default());
14068 let supervisor = SupervisorHandle::new();
14069 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14070 let handler =
14071 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14072 .with_supervisor(supervisor);
14073 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
14074 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
14075 register_capability_manifest(
14076 &handler,
14077 &target_ctx,
14078 &mut target_rx,
14079 capability_manifest("target", &["credentials-provider/v1"], &[]),
14080 1,
14081 )
14082 .await;
14083 register_capability_manifest(
14084 &handler,
14085 &old_opener_ctx,
14086 &mut old_opener_rx,
14087 capability_manifest("opener", &[], &[]),
14088 2,
14089 )
14090 .await;
14091 let (mut client_rx, _) = open_route_for_capability_test(
14092 &handler,
14093 &target_ctx,
14094 &mut target_rx,
14095 712,
14096 3,
14097 "target",
14098 Some(ConsumerIdentity {
14099 module_id: "opener".to_string(),
14100 launch_nonce: "opener-nonce".to_string(),
14101 }),
14102 )
14103 .await;
14104 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14105
14106 handler
14107 .cleanup_connection(old_opener_ctx.connection_id)
14108 .expect("old opener registration cleans up");
14109 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
14110 register_capability_manifest(
14111 &handler,
14112 &new_opener_ctx,
14113 &mut new_opener_rx,
14114 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14115 4,
14116 )
14117 .await;
14118
14119 assert_capability_denied_push(
14120 client_rx
14121 .try_recv()
14122 .expect("HELLO deny addition must emit route.closed")
14123 .frame,
14124 "target",
14125 );
14126 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14127 assert!(matches!(
14128 target_rx.try_recv(),
14129 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14130 ));
14131 }
14132
14133 #[tokio::test]
14134 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
14135 let registry = Arc::new(Registry::default());
14136 let forwarding = Arc::new(ForwardingTable::default());
14137 let supervisor = SupervisorHandle::new();
14138 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14139 let handler =
14140 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14141 .with_supervisor(supervisor);
14142 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
14143 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
14144 register_capability_manifest(
14145 &handler,
14146 &target_ctx,
14147 &mut target_rx,
14148 capability_manifest("target", &[], &[]),
14149 1,
14150 )
14151 .await;
14152 register_capability_manifest(
14153 &handler,
14154 &opener_ctx,
14155 &mut opener_rx,
14156 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14157 2,
14158 )
14159 .await;
14160 let (mut client_rx, _) = open_route_for_capability_test(
14161 &handler,
14162 &target_ctx,
14163 &mut target_rx,
14164 722,
14165 3,
14166 "target",
14167 Some(ConsumerIdentity {
14168 module_id: "opener".to_string(),
14169 launch_nonce: "opener-nonce".to_string(),
14170 }),
14171 )
14172 .await;
14173 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14174
14175 let replies = handler
14176 .handle_control_frame(
14177 &target_ctx,
14178 catalog_update_with_capabilities_frame(
14179 4,
14180 CapabilityDeclarations {
14181 provides: vec!["credentials-provider/v1".to_string()],
14182 requires: Vec::new(),
14183 must_never_reach: Vec::new(),
14184 },
14185 ),
14186 )
14187 .await
14188 .expect("claim catalog.update succeeds");
14189 assert!(matches!(
14190 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
14191 Ok(ModuleControlResponseToModule::CatalogUpdate {})
14192 ));
14193 assert_capability_denied_push(
14194 client_rx
14195 .try_recv()
14196 .expect("claim addition must emit route.closed")
14197 .frame,
14198 "target",
14199 );
14200 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14201 assert!(matches!(
14202 target_rx.try_recv(),
14203 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14204 ));
14205 }
14206
14207 #[tokio::test]
14208 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
14209 let registry = Arc::new(Registry::default());
14210 let forwarding = Arc::new(ForwardingTable::default());
14211 let supervisor = SupervisorHandle::new();
14212 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14213 let handler =
14214 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14215 .with_supervisor(supervisor);
14216 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
14217 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
14218 register_capability_manifest(
14219 &handler,
14220 &target_ctx,
14221 &mut target_rx,
14222 capability_manifest("target", &["credentials-provider/v1"], &[]),
14223 1,
14224 )
14225 .await;
14226 register_capability_manifest(
14227 &handler,
14228 &opener_ctx,
14229 &mut opener_rx,
14230 capability_manifest("opener", &[], &[]),
14231 2,
14232 )
14233 .await;
14234 let (mut client_rx, _) = open_route_for_capability_test(
14235 &handler,
14236 &target_ctx,
14237 &mut target_rx,
14238 732,
14239 3,
14240 "target",
14241 Some(ConsumerIdentity {
14242 module_id: "opener".to_string(),
14243 launch_nonce: "opener-nonce".to_string(),
14244 }),
14245 )
14246 .await;
14247 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14248
14249 handler
14250 .handle_control_frame(
14251 &target_ctx,
14252 catalog_update_with_capabilities_frame(
14253 4,
14254 CapabilityDeclarations {
14255 provides: Vec::new(),
14256 requires: Vec::new(),
14257 must_never_reach: Vec::new(),
14258 },
14259 ),
14260 )
14261 .await
14262 .expect("claim removal catalog.update succeeds");
14263 assert_eq!(
14264 forwarding.active_binding_count().unwrap(),
14265 1,
14266 "removing an attested target claim must leave the route census unchanged"
14267 );
14268 assert!(
14269 client_rx.try_recv().is_err(),
14270 "claim removal must not emit route.closed capability_denied"
14271 );
14272 assert!(
14273 target_rx.try_recv().is_err(),
14274 "claim removal must not send the target a route GOODBYE"
14275 );
14276 }
14277
14278 #[tokio::test]
14281 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
14282 let registry = Arc::new(Registry::default());
14283 let forwarding = Arc::new(ForwardingTable::default());
14284 let supervisor = SupervisorHandle::new();
14285 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14286 let handler =
14287 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14288 .with_supervisor(supervisor);
14289 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
14290 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
14291 register_capability_manifest(
14292 &handler,
14293 &target_ctx,
14294 &mut target_rx,
14295 capability_manifest("target", &["credentials-provider/v1"], &[]),
14296 1,
14297 )
14298 .await;
14299 register_capability_manifest(
14300 &handler,
14301 &opener_ctx,
14302 &mut opener_rx,
14303 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14304 2,
14305 )
14306 .await;
14307
14308 let (_client_rx, bind) = open_route_for_capability_test(
14309 &handler,
14310 &target_ctx,
14311 &mut target_rx,
14312 742,
14313 3,
14314 "target",
14315 None,
14316 )
14317 .await;
14318 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14319 panic!("direct scope-honesty route must bind");
14320 };
14321 assert_eq!(principal, Some(Principal::Direct));
14322 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14323 }
14324
14325 #[tokio::test]
14328 async fn must_never_reach_self_route_is_capability_forbidden() {
14329 let registry = Arc::new(Registry::default());
14330 let forwarding = Arc::new(ForwardingTable::default());
14331 let supervisor = SupervisorHandle::new();
14332 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
14333 let handler =
14334 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14335 .with_supervisor(supervisor);
14336 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
14337 register_capability_manifest(
14338 &handler,
14339 &self_ctx,
14340 &mut self_rx,
14341 capability_manifest(
14342 "self-provider",
14343 &["credentials-provider/v1"],
14344 &["credentials-provider/v1"],
14345 ),
14346 1,
14347 )
14348 .await;
14349
14350 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
14351 let replies = handler
14352 .handle_control_frame(
14353 &client_ctx,
14354 route_open_frame_with_admission_facts(
14355 2,
14356 "self-provider",
14357 unique_project_root("admission-facts"),
14358 Some(ConsumerIdentity {
14359 module_id: "self-provider".to_string(),
14360 launch_nonce: "self-nonce".to_string(),
14361 }),
14362 None,
14363 ),
14364 )
14365 .await
14366 .expect("self-route refusal returns a typed frame");
14367 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14368 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14369 assert!(
14370 self_rx.try_recv().is_err(),
14371 "self denial must not relay route.bind"
14372 );
14373 }
14374
14375 #[test]
14376 fn unsupported_channel_zero_frame_returns_error() {
14377 let handler = ControlHandler::default();
14378 let request = Frame::build(
14379 FrameType::Request,
14380 control_flags(),
14381 0,
14382 0,
14383 21,
14384 b"opaque".to_vec(),
14385 )
14386 .unwrap();
14387
14388 let response = handler
14389 .handle_control(ConnectionId::new(1), request)
14390 .unwrap();
14391
14392 assert_eq!(response[0].header.ty, FrameType::Error);
14393 assert_eq!(
14394 parse_error(&response[0])["code"],
14395 "unsupported_control_frame"
14396 );
14397 }
14398
14399 mod swap {
14404 use super::*;
14405
14406 const INCUMBENT: ConnectionId = ConnectionId::new(30);
14407 const CANDIDATE: ConnectionId = ConnectionId::new(40);
14408
14409 struct Swap {
14410 registry: Arc<Registry>,
14411 forwarding: Arc<ForwardingTable>,
14412 handler: ControlHandler,
14413 incumbent_ctx: RouteCtx,
14414 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14415 candidate_ctx: RouteCtx,
14416 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14417 }
14418
14419 async fn swap_with_incumbent() -> Swap {
14420 let registry = Arc::new(Registry::default());
14421 let forwarding = Arc::new(ForwardingTable::default());
14422 let handler =
14423 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14424 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
14425 hello_via_sink(
14426 &handler,
14427 &incumbent_ctx,
14428 &mut incumbent_rx,
14429 hello_frame("aft", PROTOCOL_VERSION, 7),
14430 )
14431 .await;
14432 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
14433 Swap {
14434 registry,
14435 forwarding,
14436 handler,
14437 incumbent_ctx,
14438 incumbent_rx,
14439 candidate_ctx,
14440 candidate_rx,
14441 }
14442 }
14443
14444 fn register_candidate(swap: &Swap, ready: Option<bool>) {
14445 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
14446 candidate_manifest.ready = ready;
14447 let registration = swap
14448 .registry
14449 .register_candidate_with_control_ops(
14450 candidate_manifest,
14451 PROTOCOL_VERSION,
14452 CANDIDATE,
14453 module_baseline_control_ops(),
14454 )
14455 .unwrap();
14456 swap.forwarding
14457 .register_candidate_module_connection(
14458 CANDIDATE,
14459 "aft".to_string(),
14460 PROTOCOL_VERSION,
14461 manifest_concurrency(®istration.manifest),
14462 swap.candidate_ctx.egress.clone(),
14463 )
14464 .unwrap();
14465 }
14466
14467 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
14468 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
14469 swap.registry.promote_candidate("aft").unwrap().unwrap();
14470 cutover.incumbent.unwrap()
14471 }
14472
14473 #[tokio::test]
14482 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
14483 let mut swap = swap_with_incumbent().await;
14484 let hold = Arc::new(std::sync::Barrier::new(2));
14485 let handler = swap
14486 .handler
14487 .clone()
14488 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14489 handler
14490 .supervisor
14491 .open_swap("aft", "candidate-nonce".to_string());
14492 let hello = {
14493 let handler = handler.clone();
14494 let candidate_ctx = swap.candidate_ctx.clone();
14495 std::thread::spawn(move || {
14496 handler
14497 .handle_hello(
14498 candidate_ctx.connection_id,
14499 Some(candidate_ctx.egress.clone()),
14500 hello_frame_with_nonce(
14501 "aft",
14502 PROTOCOL_VERSION,
14503 8,
14504 Some("candidate-nonce"),
14505 ),
14506 )
14507 .unwrap()
14508 })
14509 };
14510
14511 hold.wait();
14512 assert!(swap
14515 .registry
14516 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
14517 .unwrap()
14518 .is_some());
14519 assert!(swap
14520 .registry
14521 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14522 .unwrap()
14523 .is_none());
14524 assert!(swap
14525 .forwarding
14526 .module_endpoint_for_connection(CANDIDATE)
14527 .unwrap()
14528 .is_none());
14529
14530 hold.wait();
14531 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
14532 assert_eq!(
14533 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
14534 FrameType::HelloAck
14535 );
14536 let published = swap
14537 .registry
14538 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14539 .unwrap()
14540 .expect("candidate is published once its HELLO completes");
14541 assert_eq!(published.connection_id, CANDIDATE);
14542 assert!(swap
14543 .forwarding
14544 .module_endpoint_for_connection(CANDIDATE)
14545 .unwrap()
14546 .is_some());
14547 }
14548
14549 fn keyed_total(counters: &Value, key: &str) -> u64 {
14550 counters[key]
14551 .as_object()
14552 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
14553 .unwrap_or(0)
14554 }
14555
14556 #[tokio::test]
14557 async fn replacement_logs_old_end_and_new_admission_once() {
14558 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14559 let mut swap = swap_with_incumbent().await;
14560 swap.handler
14561 .supervisor
14562 .open_swap("aft", "candidate-nonce".to_string());
14563 hello_via_sink(
14564 &swap.handler,
14565 &swap.candidate_ctx,
14566 &mut swap.candidate_rx,
14567 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
14568 )
14569 .await;
14570 cutover(&swap);
14571 swap.handler.cleanup_connection(INCUMBENT).unwrap();
14572
14573 let captured = crate::router::test_log::captured_logs(&logs);
14574 println!("captured replacement registry logs:\n{captured}");
14575 let new_admissions: Vec<_> = captured
14576 .lines()
14577 .filter(|line| {
14578 line.contains("connection_id=40")
14579 && line.contains("swap candidate registered; not routable until cutover")
14580 })
14581 .collect();
14582 assert_eq!(
14583 new_admissions.len(),
14584 1,
14585 "unexpected admission logs: {captured}"
14586 );
14587 assert!(new_admissions[0].contains("module_id=aft"));
14588 assert!(new_admissions[0].contains("connection_id=40"));
14589 assert!(
14590 new_admissions[0].contains("swap candidate registered; not routable until cutover")
14591 );
14592 assert!(new_admissions[0].contains("ready=true"));
14593
14594 let old_admissions: Vec<_> = captured
14595 .lines()
14596 .filter(|line| {
14597 line.contains("module registered module_id=aft ")
14598 && line.contains("connection_id=30")
14599 })
14600 .collect();
14601 assert_eq!(
14602 old_admissions.len(),
14603 1,
14604 "unexpected incumbent admission: {captured}"
14605 );
14606
14607 let old_ends: Vec<_> = captured
14608 .lines()
14609 .filter(|line| {
14610 line.contains("connection_id=30") && line.contains("module registration ended")
14611 })
14612 .collect();
14613 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
14614 assert!(old_ends[0].contains("module_id=aft"));
14615 assert!(old_ends[0].contains("connection_id=30"));
14616 assert!(old_ends[0].contains("reason=replaced"));
14617 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
14618 assert_eq!(
14619 captured
14620 .lines()
14621 .filter(|line| line.contains("module registration promoted"))
14622 .count(),
14623 1,
14624 "unexpected promotion logs: {captured}"
14625 );
14626 }
14627
14628 #[tokio::test]
14634 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
14635 let mut swap = swap_with_incumbent().await;
14636 let handler = swap.handler.clone();
14637
14638 let cotenant = ConnectionId::new(31);
14640 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
14641 let (cotenant_task, cotenant_bind) = relay_route_open(
14642 &handler,
14643 cotenant,
14644 &cotenant_ctx.egress,
14645 &mut swap.incumbent_rx,
14646 100,
14647 "aft",
14648 "swap-cotenant",
14649 )
14650 .await;
14651 handler
14652 .handle_control_frame(
14653 &swap.incumbent_ctx,
14654 route_bind_ack(cotenant_bind.header.corr),
14655 )
14656 .await
14657 .unwrap();
14658 assert!(cotenant_task.await.unwrap().is_empty());
14659 let (cotenant_channel, cotenant_epoch) =
14660 published_route(&cotenant_rx.recv().await.unwrap());
14661
14662 let caller = ConnectionId::new(32);
14664 let (caller_ctx, mut caller_rx) = route_ctx(caller);
14665 let (caller_task, caller_bind) = relay_route_open(
14666 &handler,
14667 caller,
14668 &caller_ctx.egress,
14669 &mut swap.incumbent_rx,
14670 101,
14671 "aft",
14672 "swap-caller",
14673 )
14674 .await;
14675 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
14676
14677 register_candidate(&swap, None);
14678 cutover(&swap);
14679
14680 let ack = handler
14682 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
14683 .await;
14684 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
14685 if module_loop_error.is_some() {
14686 handler.cleanup_connection(INCUMBENT).unwrap();
14689 }
14690
14691 assert!(
14693 cotenant_rx.try_recv().is_err(),
14694 "the co-tenant route on the incumbent was torn down by one late ack: \
14695 {module_loop_error:?}"
14696 );
14697 assert!(matches!(
14698 swap.forwarding
14699 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14700 .unwrap(),
14701 DataRoute::Client(DataRouteState::Bound(_))
14702 ));
14703 assert_eq!(module_loop_error, None);
14704 assert!(swap
14705 .registry
14706 .get_module_by_connection(INCUMBENT)
14707 .unwrap()
14708 .is_some());
14709
14710 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
14712 .await
14713 .expect("the incumbent is told to drop the abandoned binding")
14714 .unwrap()
14715 .frame;
14716 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14717 assert_eq!(goodbye.header.channel, abandoned_channel);
14718 assert_eq!(goodbye.header.epoch, abandoned_epoch);
14719 assert!(swap.incumbent_rx.try_recv().is_err());
14720
14721 let response = caller_task.await.unwrap();
14723 assert_eq!(response.len(), 1);
14724 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14725 assert!(caller_rx.try_recv().is_err());
14726
14727 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
14731 let counters = handler.counters().snapshot();
14732 assert_eq!(
14733 keyed_total(&counters, "route_open_accepted_by_principal"),
14734 1
14735 );
14736 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
14737 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
14738 }
14739
14740 #[tokio::test]
14744 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
14745 let mut swap = swap_with_incumbent().await;
14746 register_candidate(&swap, None);
14747 let incumbent = cutover(&swap);
14748 swap.forwarding
14749 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
14750 .unwrap()
14751 .expect("the incumbent is still registered");
14752
14753 let client = ConnectionId::new(33);
14754 let (client_ctx, mut client_rx) = route_ctx(client);
14755 let route_handler = swap.handler.clone();
14756 let open_ctx = RouteCtx {
14757 connection_id: client,
14758 egress: client_ctx.egress.clone(),
14759 };
14760 let mut route_task = tokio::spawn(async move {
14761 route_handler
14762 .handle_control_frame(
14763 &open_ctx,
14764 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
14765 )
14766 .await
14767 .unwrap()
14768 });
14769 let bind = tokio::select! {
14770 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
14771 response = &mut route_task => {
14772 let response = response.unwrap();
14773 panic!(
14774 "post-cutover route.open was refused instead of relayed to the candidate: {}",
14775 parse_error(&response[0])["code"]
14776 );
14777 }
14778 };
14779 swap.handler
14780 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
14781 .await
14782 .unwrap();
14783 assert!(route_task.await.unwrap().is_empty());
14784 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
14785 match swap
14786 .forwarding
14787 .lookup_data_route(client, channel, epoch)
14788 .unwrap()
14789 {
14790 DataRoute::Client(DataRouteState::Bound(route)) => {
14791 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
14792 }
14793 other => panic!("expected a bound route on the candidate, got {other:?}"),
14794 }
14795 assert!(swap.incumbent_rx.try_recv().is_err());
14796 }
14797
14798 #[tokio::test]
14803 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
14804 let swap = swap_with_incumbent().await;
14805 register_candidate(&swap, Some(false));
14806 let update = Frame::build(
14807 FrameType::Request,
14808 control_flags(),
14809 0,
14810 0,
14811 55,
14812 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14813 provides: manifest("aft", PROTOCOL_VERSION).provides,
14814 capabilities: None,
14815 ready: Some(true),
14816 })
14817 .unwrap(),
14818 )
14819 .unwrap();
14820
14821 let replies = swap
14822 .handler
14823 .handle_control_frame(&swap.candidate_ctx, update)
14824 .await
14825 .unwrap();
14826
14827 assert_eq!(replies.len(), 1);
14828 assert_eq!(
14829 replies[0].header.ty,
14830 FrameType::Response,
14831 "candidate catalog.update was refused: {:?}",
14832 serde_json::from_slice::<Value>(&replies[0].body).ok()
14833 );
14834 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
14835 assert_eq!(
14836 swap.registry
14837 .get_module("aft")
14838 .unwrap()
14839 .unwrap()
14840 .connection_id,
14841 INCUMBENT
14842 );
14843 }
14844 }
14845
14846 mod swap_admission {
14850 use super::*;
14851
14852 const INCUMBENT_NONCE: &str = "incumbent-nonce";
14853 const CANDIDATE_NONCE: &str = "candidate-nonce";
14854
14855 fn handler_with_incumbent(
14856 module_id: &str,
14857 reserved: bool,
14858 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
14859 let registry = Arc::new(Registry::default());
14860 let supervisor = SupervisorHandle::new();
14861 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
14862 if reserved {
14863 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
14864 }
14865 let handler =
14866 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
14867 let incumbent = handler
14868 .handle_control(
14869 ConnectionId::new(1),
14870 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
14871 )
14872 .unwrap();
14873 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
14874 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
14875 (registry, supervisor, handler)
14876 }
14877
14878 #[test]
14884 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
14885 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
14886
14887 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
14888 let replies = handler
14889 .handle_control(
14890 ConnectionId::new(connection),
14891 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
14892 )
14893 .unwrap();
14894 assert_eq!(replies[0].header.ty, FrameType::Error);
14895 assert_eq!(
14896 parse_error(&replies[0])["code"],
14897 "swap_token_invalid",
14898 "nonce {nonce:?}"
14899 );
14900 }
14901 assert!(registry.get_candidate("aft").unwrap().is_none());
14902 assert_eq!(
14903 registry.get_module("aft").unwrap().unwrap().connection_id,
14904 ConnectionId::new(1)
14905 );
14906
14907 let admitted = handler
14910 .handle_control(
14911 ConnectionId::new(4),
14912 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
14913 )
14914 .unwrap();
14915 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
14916 assert_eq!(
14917 registry
14918 .get_candidate("aft")
14919 .unwrap()
14920 .unwrap()
14921 .connection_id,
14922 ConnectionId::new(4)
14923 );
14924 assert_eq!(
14925 registry.get_module("aft").unwrap().unwrap().connection_id,
14926 ConnectionId::new(1),
14927 "the candidate must not take the active slot"
14928 );
14929 let replayed = handler
14930 .handle_control(
14931 ConnectionId::new(5),
14932 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
14933 )
14934 .unwrap();
14935 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
14936
14937 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
14941 let squatter = handler
14942 .handle_control(
14943 ConnectionId::new(6),
14944 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
14945 )
14946 .unwrap();
14947 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
14948 assert!(
14949 registry.get_module("aft").unwrap().is_none(),
14950 "a squatter took the active slot of an id being swapped"
14951 );
14952 }
14953
14954 #[test]
14959 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
14960 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
14961
14962 let replies = handler
14963 .handle_control(
14964 ConnectionId::new(2),
14965 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14966 )
14967 .unwrap();
14968
14969 assert_eq!(
14970 replies[0].header.ty,
14971 FrameType::HelloAck,
14972 "reserved candidate refused: {:?}",
14973 serde_json::from_slice::<Value>(&replies[0].body).ok()
14974 );
14975 assert_eq!(
14976 registry
14977 .get_candidate("vault")
14978 .unwrap()
14979 .unwrap()
14980 .connection_id,
14981 ConnectionId::new(2)
14982 );
14983 }
14984
14985 #[test]
14988 fn without_an_open_swap_the_ordinary_gates_decide() {
14989 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
14990 supervisor.close_swap("vault");
14991
14992 let candidate = handler
14993 .handle_control(
14994 ConnectionId::new(2),
14995 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14996 )
14997 .unwrap();
14998 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
14999 let duplicate = handler
15000 .handle_control(
15001 ConnectionId::new(3),
15002 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
15003 )
15004 .unwrap();
15005 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15006 assert!(registry.get_candidate("vault").unwrap().is_none());
15007 }
15008 }
15009
15010 mod scopes {
15014 mod idle_tests {
15015 use super::*;
15016 include!("scope_idle_tests.rs");
15017 }
15018 mod route_open_timings {
15019 use super::*;
15020 include!("route_open_timing_tests.rs");
15021 }
15022 mod activity {
15023 include!("activity_scope_tests.rs");
15024 }
15025 mod delta {
15026 include!("scope_handler_delta_tests.rs");
15027 }
15028 mod call_modes {
15029 include!("scope_call_mode_tests.rs");
15030 }
15031 #[tokio::test]
15032 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
15033 let rig = rig().await;
15034 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
15035 "s".into(),
15036 1,
15037 "tool".into(),
15038 ))
15039 .unwrap();
15040 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
15041 let (direct, _rx) = wide_ctx(999);
15042 for (ctx, expected) in [
15043 (&rig.owner, FrameType::Response),
15044 (&direct, FrameType::Error),
15045 ] {
15046 let frame =
15047 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
15048 .unwrap();
15049 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
15050 assert_eq!(replies[0].header.ty, expected);
15051 if expected == FrameType::Response {
15052 let reply: subc_control::ScopeServedByReply =
15053 serde_json::from_slice(&replies[0].body).unwrap();
15054 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
15055 } else {
15056 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
15057 }
15058 }
15059 let reply = call(
15060 &rig.handler,
15061 &rig.owner,
15062 &ModuleControlRequestFromModule::ScopeDescribe {
15063 owner: Principal::Reserved {
15064 module_id: OWNER.into(),
15065 },
15066 scope_ref: "s".into(),
15067 },
15068 )
15069 .await;
15070 assert!(matches!(
15071 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
15072 ModuleControlResponseToModule::ScopeDescribe { .. }
15073 ));
15074 }
15075
15076 use subc_protocol::scope::{
15077 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
15078 ScopeStatus,
15079 };
15080
15081 use super::*;
15082
15083 const OWNER: &str = "prefrontal-core";
15084
15085 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
15086 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
15087 }
15088
15089 async fn call(
15090 handler: &ControlHandler,
15091 ctx: &RouteCtx,
15092 request: &ModuleControlRequestFromModule,
15093 ) -> Frame {
15094 let body = serde_json::to_vec(request).unwrap();
15095 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
15096 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
15097 assert_eq!(replies.len(), 1, "{replies:?}");
15098 replies.pop().unwrap()
15099 }
15100
15101 async fn sync(
15102 handler: &ControlHandler,
15103 ctx: &RouteCtx,
15104 generation: u64,
15105 scopes: Vec<ScopeRecord>,
15106 ) -> Result<ModuleControlResponseToModule, String> {
15107 let reply = call(
15108 handler,
15109 ctx,
15110 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
15111 )
15112 .await;
15113 match reply.header.ty {
15114 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
15115 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
15116 }
15117 }
15118
15119 async fn describe(
15120 handler: &ControlHandler,
15121 ctx: &RouteCtx,
15122 owner: &str,
15123 scope_ref: &str,
15124 ) -> ModuleControlResponseToModule {
15125 let reply = call(
15126 handler,
15127 ctx,
15128 &ModuleControlRequestFromModule::ScopeDescribe {
15129 owner: Principal::Reserved {
15130 module_id: owner.to_string(),
15131 },
15132 scope_ref: scope_ref.to_string(),
15133 },
15134 )
15135 .await;
15136 assert_eq!(
15137 reply.header.ty,
15138 FrameType::Response,
15139 "{:?}",
15140 parse_error(&reply)
15141 );
15142 serde_json::from_slice(&reply.body).unwrap()
15143 }
15144
15145 #[tokio::test]
15146 async fn scope_describe_never_echoes_bind_proof_or_hash() {
15147 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15148 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15149 let mut rig = rig().await;
15150 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15151 .await;
15152 let _bound = rig
15153 .bound(
15154 Some(AFT),
15155 PLEXUS,
15156 Some(
15157 rig.selector("s", Some(1))
15158 .with_bind_proof(Some(PROOF.into())),
15159 ),
15160 )
15161 .await;
15162 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
15163 let output = serde_json::to_string(&reply).unwrap();
15164 assert!(output.contains("\"status\":\"live\""));
15165 assert!(!output.contains(PROOF));
15166 assert!(!output.contains(HASH));
15167 }
15168
15169 async fn module(
15171 handler: &ControlHandler,
15172 connection: u64,
15173 module_id: &str,
15174 nonce: Option<&str>,
15175 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15176 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
15177 hello_via_sink(
15178 handler,
15179 &ctx,
15180 &mut rx,
15181 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
15182 )
15183 .await;
15184 (ctx, rx)
15185 }
15186
15187 #[tokio::test]
15190 async fn a_client_connection_cannot_sync_or_describe() {
15191 let handler = ControlHandler::new(Arc::new(Registry::default()));
15192 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
15193 for request in [
15194 ModuleControlRequestFromModule::ScopeSync {
15195 generation: 1,
15196 scopes: vec![head("s", 1)],
15197 },
15198 ModuleControlRequestFromModule::ScopeDescribe {
15199 owner: Principal::Direct,
15200 scope_ref: "s".to_string(),
15201 },
15202 ] {
15203 let reply = call(&handler, &ctx, &request).await;
15204 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
15205 }
15206 assert!(
15207 !handler
15208 .scopes
15209 .read()
15210 .unwrap()
15211 .describe(
15212 &Principal::Reserved {
15213 module_id: OWNER.to_string()
15214 },
15215 "s"
15216 )
15217 .owner_synced
15218 );
15219 }
15220
15221 #[tokio::test]
15224 async fn a_module_without_a_supervised_launch_cannot_sync() {
15225 let handler = ControlHandler::new(Arc::new(Registry::default()));
15226 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
15227 assert_eq!(
15228 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
15229 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15230 );
15231 }
15232
15233 #[tokio::test]
15234 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
15235 let supervisor = SupervisorHandle::new();
15236 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15237 let handler = ControlHandler::new(Arc::new(Registry::default()))
15238 .with_supervisor(supervisor.clone());
15239 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15240 sync(&handler, &incumbent, 1, vec![head("s", 1)])
15241 .await
15242 .expect("the current launch syncs");
15243
15244 supervisor.open_swap(OWNER, "n2".to_string());
15247 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
15248 assert_eq!(
15249 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15250 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15251 );
15252 sync(&handler, &incumbent, 2, vec![head("s", 1)])
15253 .await
15254 .expect("the serving owner syncs during the swap");
15255
15256 supervisor.close_swap(OWNER);
15259 assert_eq!(
15260 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15261 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15262 );
15263 sync(&handler, &incumbent, 3, vec![head("s", 1)])
15264 .await
15265 .expect("the serving owner syncs after the rollback");
15266 handler.cleanup_connection(candidate.connection_id).unwrap();
15267
15268 supervisor.open_swap(OWNER, "n3".to_string());
15273 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
15274 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
15275 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
15276 .await
15277 .expect("the promoted launch takes authority");
15278 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15279 panic!("unexpected reply {reply:?}");
15280 };
15281 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
15282 assert_eq!(
15283 sync(&handler, &incumbent, 4, Vec::new()).await,
15284 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15285 );
15286 }
15287
15288 #[tokio::test]
15291 async fn closing_the_authority_connection_frees_sync_authority() {
15292 let supervisor = SupervisorHandle::new();
15293 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15294 let handler = ControlHandler::new(Arc::new(Registry::default()))
15295 .with_supervisor(supervisor.clone());
15296 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15297 sync(&handler, &first, 10, vec![head("s", 1)])
15298 .await
15299 .unwrap();
15300 handler.cleanup_connection(first.connection_id).unwrap();
15301
15302 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
15303 sync(&handler, &second, 1, vec![head("s", 1)])
15304 .await
15305 .expect("the next connection takes the released authority");
15306 }
15307
15308 #[tokio::test]
15309 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
15310 let supervisor = SupervisorHandle::new();
15311 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15312 let handler =
15313 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
15314 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
15315 sync(&handler, &first, 10, vec![head("s", 1)])
15316 .await
15317 .unwrap();
15318 handler
15319 .handle_control_frame(
15320 &first,
15321 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
15322 )
15323 .await
15324 .unwrap();
15325 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
15326 sync(&handler, &second, 1, vec![head("s", 1)])
15327 .await
15328 .expect("GOODBYE releases authority even if the old socket remains open");
15329 }
15330
15331 #[tokio::test]
15332 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
15333 let registry = Arc::new(Registry::default());
15334 let supervisor_handle = SupervisorHandle::new();
15335 let supervisor =
15336 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15337 .with_handle(supervisor_handle.clone())
15338 .with_daemon_incarnation("incarnation-7".to_string());
15339 supervisor
15342 .supervise_configured(
15343 ModuleSpec {
15344 module_id: OWNER.to_string(),
15345 program: PathBuf::from("/nonexistent/prefrontal-core"),
15346 args: Vec::new(),
15347 env: Vec::new(),
15348 reserved: false,
15349 reserved_prefixes: Vec::new(),
15350 protocol: ModuleProtocol::Subc,
15351 overlap: Default::default(),
15352 },
15353 false,
15354 )
15355 .unwrap();
15356 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
15357 let handler =
15358 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15359 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
15360
15361 let ModuleControlResponseToModule::ScopeDescribe {
15363 status,
15364 daemon_incarnation,
15365 owner_synced,
15366 owner_configured,
15367 scope,
15368 ..
15369 } = describe(&handler, &reader, OWNER, "s").await
15370 else {
15371 panic!("not a describe reply");
15372 };
15373 assert_eq!(status, ScopeStatus::NotLive);
15374 assert_eq!(daemon_incarnation, "incarnation-7");
15375 assert!(!owner_synced);
15376 assert!(owner_configured);
15377 assert!(scope.is_none());
15378
15379 let ModuleControlResponseToModule::ScopeDescribe {
15382 status,
15383 owner_configured,
15384 ..
15385 } = describe(&handler, &reader, "ghost", "s").await
15386 else {
15387 panic!("not a describe reply");
15388 };
15389 assert_eq!(status, ScopeStatus::NotLive);
15390 assert!(!owner_configured);
15391
15392 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
15394 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
15395 let ModuleControlResponseToModule::ScopeDescribe {
15396 status,
15397 scope_epoch,
15398 owner_synced,
15399 scope,
15400 ..
15401 } = describe(&handler, &reader, OWNER, "s").await
15402 else {
15403 panic!("not a describe reply");
15404 };
15405 assert_eq!(status, ScopeStatus::Live);
15406 assert_eq!(scope_epoch, Some(4));
15407 assert!(owner_synced);
15408 let stamp = scope.expect("a live scope carries its stamp");
15409 assert!(
15410 stamp.owner_authorized,
15411 "prefrontal-core is the default authority"
15412 );
15413 assert_eq!(stamp.kind, ScopeKind::Head);
15414 }
15415
15416 #[tokio::test]
15417 async fn scope_authority_owners_decides_owner_authorized() {
15418 let supervisor = SupervisorHandle::new();
15419 supervisor.set_spawn_nonce("broca", "b1".to_string());
15420 let handler = ControlHandler::new(Arc::new(Registry::default()))
15421 .with_supervisor(supervisor)
15422 .with_scope_authority_owners(vec!["broca".to_string()]);
15423 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
15424 let mut gated = head("s", 1);
15425 gated.attributes.agent_id = Some("agent".to_string());
15426 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
15427 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
15428 describe(&handler, &broca, "broca", "s").await
15429 else {
15430 panic!("not a describe reply");
15431 };
15432 assert!(scope.unwrap().owner_authorized);
15433 }
15434
15435 #[tokio::test]
15439 async fn scope_ops_and_the_scopes_capability_are_advertised() {
15440 let handler = ControlHandler::new(Arc::new(Registry::default()));
15441 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
15442 let ack = hello_via_sink(
15443 &handler,
15444 &ctx,
15445 &mut rx,
15446 hello_frame("m", PROTOCOL_VERSION, 1),
15447 )
15448 .await;
15449 let ack = parse_ack(&ack);
15450 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
15451 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
15452 }
15453 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
15454
15455 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
15456 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15457 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15458 let reply = handler
15459 .handle_control_frame(&client, frame)
15460 .await
15461 .unwrap()
15462 .pop()
15463 .unwrap();
15464 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15465 serde_json::from_slice(&reply.body).unwrap()
15466 else {
15467 panic!("not a server.describe reply");
15468 };
15469 assert!(
15470 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
15471 "{capabilities:?}"
15472 );
15473 }
15474
15475 const PLEXUS: &str = "plexus";
15478 const OTHER: &str = "other";
15479 const AFT: &str = "aft";
15480 const BROCA: &str = "broca";
15481 const MAGIC: &str = "magic-context";
15482
15483 fn nonce(module_id: &str) -> String {
15484 format!("nonce-{module_id}")
15485 }
15486
15487 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15488 let (tx, rx) = mpsc::channel(64);
15489 (
15490 RouteCtx {
15491 connection_id: ConnectionId::new(connection),
15492 egress: FrameSink::new(tx),
15493 },
15494 rx,
15495 )
15496 }
15497
15498 struct Rig {
15502 handler: ControlHandler,
15503 forwarding: Arc<ForwardingTable>,
15504 owner: RouteCtx,
15505 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15506 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
15507 generation: u64,
15508 next_connection: u64,
15509 _supervisor: Supervisor,
15510 }
15511
15512 async fn rig() -> Rig {
15513 rig_with_flow_support(true).await
15514 }
15515
15516 async fn rig_with_flow_support(flow_support: bool) -> Rig {
15517 rig_with_scope_capabilities(flow_support, false).await
15518 }
15519
15520 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
15521 let mut provides = Vec::new();
15522 if flow_support {
15523 provides.push("flow-scopes/v1");
15524 }
15525 if run_support {
15526 provides.push("agent-run-scopes/v1");
15527 }
15528 rig_with_target_capabilities(&provides).await
15529 }
15530
15531 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
15532 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
15533 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15534 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15537 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
15538 Frame::build(
15539 FrameType::Hello,
15540 control_flags(),
15541 0,
15542 0,
15543 connection,
15544 serde_json::to_vec(&body).unwrap(),
15545 )
15546 .unwrap()
15547 }
15548
15549 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
15550 let registry = Arc::new(Registry::default());
15551 let forwarding = Arc::new(ForwardingTable::default());
15552 let supervisor_handle = SupervisorHandle::new();
15553 let supervisor =
15554 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15555 .with_handle(supervisor_handle.clone());
15556 supervisor
15557 .supervise_configured(
15558 ModuleSpec {
15559 module_id: OWNER.to_string(),
15560 program: PathBuf::from("/nonexistent/prefrontal-core"),
15561 args: Vec::new(),
15562 env: Vec::new(),
15563 reserved: false,
15564 reserved_prefixes: Vec::new(),
15565 protocol: ModuleProtocol::Subc,
15566 overlap: Default::default(),
15567 },
15568 false,
15569 )
15570 .unwrap();
15571 for module_id in [OWNER, AFT, BROCA, MAGIC] {
15572 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
15573 }
15574 let handler =
15575 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15576 .with_supervisor(supervisor_handle);
15577 let (owner, mut owner_rx) = wide_ctx(1);
15578 hello_via_sink(
15579 &handler,
15580 &owner,
15581 &mut owner_rx,
15582 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
15583 )
15584 .await;
15585 let mut modules = BTreeMap::new();
15586 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
15587 let (ctx, mut rx) = wide_ctx(connection);
15588 let hello = hello_with_capabilities(module_id, connection, provides);
15589 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
15590 modules.insert(module_id.to_string(), (ctx, rx));
15591 }
15592 Rig {
15593 handler,
15594 forwarding,
15595 owner,
15596 _owner_rx: owner_rx,
15597 modules,
15598 generation: 0,
15599 next_connection: 100,
15600 _supervisor: supervisor,
15601 }
15602 }
15603
15604 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
15605 ScopeCarrier::new(Principal::Reserved {
15606 module_id: module_id.to_string(),
15607 })
15608 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
15609 }
15610
15611 fn session(scope_epoch: u64) -> ScopeRecord {
15614 let mut record = head("s", scope_epoch);
15615 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
15616 record.attributes.agent_id = Some("agent-1".to_string());
15617 record.attributes.delegates = true;
15618 record
15619 }
15620
15621 impl Rig {
15622 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
15623 self.generation += 1;
15624 sync(&self.handler, &self.owner, self.generation, scopes)
15625 .await
15626 .expect("the owner's sync is accepted");
15627 }
15628
15629 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15630 ScopeSelector::new(
15631 Principal::Reserved {
15632 module_id: OWNER.to_string(),
15633 },
15634 scope_ref.to_string(),
15635 scope_epoch,
15636 )
15637 }
15638
15639 fn open_frame(
15640 &mut self,
15641 opener: Option<&str>,
15642 target: &str,
15643 scope: Option<ScopeSelector>,
15644 ) -> (
15645 RouteCtx,
15646 mpsc::Receiver<crate::router::OutboundFrame>,
15647 Frame,
15648 ) {
15649 self.next_connection += 1;
15650 let (ctx, rx) = wide_ctx(self.next_connection);
15651 let root = unique_project_root("scoped-open");
15652 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15653 target: RouteTarget::ToolProvider {
15654 module_id: target.to_string(),
15655 },
15656 identity: BindIdentity::new(
15657 root.path().to_path_buf(),
15658 "unit".to_string(),
15659 "session".to_string(),
15660 ),
15661 consumer_identity: opener.map(|module_id| ConsumerIdentity {
15662 module_id: module_id.to_string(),
15663 launch_nonce: nonce(module_id),
15664 }),
15665 consumer_capabilities: None,
15666 role_versions: None,
15667 admission_facts: None,
15668 scope,
15669 })
15670 .unwrap();
15671 let frame = Frame::build(
15672 FrameType::Request,
15673 control_flags(),
15674 0,
15675 0,
15676 self.next_connection,
15677 body,
15678 )
15679 .unwrap();
15680 (ctx, rx, frame)
15681 }
15682
15683 async fn refused(
15685 &mut self,
15686 opener: Option<&str>,
15687 target: &str,
15688 scope: Option<ScopeSelector>,
15689 ) -> String {
15690 self.refusal_body(opener, target, scope).await["code"]
15691 .as_str()
15692 .unwrap()
15693 .to_string()
15694 }
15695
15696 async fn refusal_body(
15697 &mut self,
15698 opener: Option<&str>,
15699 target: &str,
15700 scope: Option<ScopeSelector>,
15701 ) -> Value {
15702 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
15703 let replies = tokio::time::timeout(
15704 Duration::from_secs(2),
15705 self.handler.handle_control_frame(&ctx, frame),
15706 )
15707 .await
15708 .expect("the open must be refused before waiting for a bind ack")
15709 .unwrap();
15710 assert_eq!(replies.len(), 1, "{replies:?}");
15711 assert_eq!(replies[0].header.ty, FrameType::Error);
15712 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15713 assert!(
15714 module_rx.try_recv().is_err(),
15715 "a refused open relays nothing"
15716 );
15717 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
15718 parse_error(&replies[0])
15719 }
15720
15721 async fn relayed(
15723 &mut self,
15724 opener: Option<&str>,
15725 target: &str,
15726 scope: Option<ScopeSelector>,
15727 ) -> Relayed {
15728 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
15729 let handler = self.handler.clone();
15730 let task_ctx = ctx.clone();
15731 let task = tokio::spawn(async move {
15732 handler
15733 .handle_control_frame(&task_ctx, frame)
15734 .await
15735 .unwrap()
15736 });
15737 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15738 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
15739 .await
15740 .expect("the target receives the relayed route.bind")
15741 .unwrap()
15742 .frame;
15743 Relayed {
15744 target: target.to_string(),
15745 client: ctx,
15746 client_rx: rx,
15747 task,
15748 bind,
15749 }
15750 }
15751
15752 async fn ack(&self, relayed: &Relayed) {
15753 let (module, _) = &self.modules[&relayed.target];
15754 self.handler
15755 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
15756 .await
15757 .unwrap();
15758 }
15759
15760 async fn bound(
15762 &mut self,
15763 opener: Option<&str>,
15764 target: &str,
15765 scope: Option<ScopeSelector>,
15766 ) -> Bound {
15767 let relayed = self.relayed(opener, target, scope).await;
15768 self.ack(&relayed).await;
15769 let Relayed {
15770 target,
15771 client,
15772 mut client_rx,
15773 task,
15774 bind,
15775 } = relayed;
15776 assert!(
15777 task.await.unwrap().is_empty(),
15778 "the open is answered by commit"
15779 );
15780 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
15781 Bound {
15782 target,
15783 client,
15784 client_rx,
15785 channel,
15786 epoch,
15787 bind,
15788 }
15789 }
15790
15791 fn live(&self, route: &Bound) -> bool {
15792 matches!(
15793 self.forwarding
15794 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15795 .unwrap(),
15796 DataRoute::Client(DataRouteState::Bound(_))
15797 )
15798 }
15799 }
15800
15801 struct Relayed {
15802 target: String,
15803 client: RouteCtx,
15804 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15805 task: tokio::task::JoinHandle<Vec<Frame>>,
15806 bind: Frame,
15807 }
15808
15809 struct Bound {
15810 target: String,
15811 client: RouteCtx,
15812 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15813 channel: u16,
15814 epoch: u32,
15815 bind: Frame,
15816 }
15817
15818 impl Bound {
15819 fn closed_reason(&mut self) -> RouteCloseReason {
15822 let mut reason = None;
15823 let mut goodbye = false;
15824 while let Ok(outbound) = self.client_rx.try_recv() {
15825 let frame = outbound.frame;
15826 match frame.header.ty {
15827 FrameType::Goodbye => {
15828 assert_eq!(
15829 (frame.header.channel, frame.header.epoch),
15830 (self.channel, self.epoch)
15831 );
15832 goodbye = true;
15833 }
15834 FrameType::Push => {
15835 let ClientControlPush::RouteClosed {
15836 reason: r,
15837 module_id,
15838 ..
15839 } = serde_json::from_slice(&frame.body).unwrap()
15840 else {
15841 panic!("unexpected push");
15842 };
15843 assert_eq!(module_id, self.target);
15844 reason = Some(r);
15845 }
15846 other => panic!("unexpected frame {other:?}"),
15847 }
15848 }
15849 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
15850 reason.expect("the client is told why the route closed")
15851 }
15852
15853 fn untouched(&mut self) -> bool {
15854 self.client_rx.try_recv().is_err()
15855 }
15856
15857 fn stamp(&self) -> Option<ScopeStamp> {
15858 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
15859 ModuleControlRequest::RouteBind { scope, .. } => scope,
15860 other => panic!("expected a route.bind, got {other:?}"),
15861 }
15862 }
15863 }
15864
15865 #[tokio::test]
15866 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
15867 ) {
15868 let mut rig = rig().await;
15869 let mut record = session(1);
15870 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
15871 record.child_owners = vec![Principal::Reserved {
15872 module_id: MAGIC.to_string(),
15873 }];
15874 rig.sync(vec![record]).await;
15875 let scope = || Some(rig_selector("s", Some(1)));
15876
15877 rig.bound(Some(OWNER), PLEXUS, scope()).await;
15880 rig.bound(Some(AFT), OTHER, scope()).await;
15881 rig.bound(Some(BROCA), PLEXUS, scope()).await;
15882
15883 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
15887 assert_eq!(
15888 rig.refused(opener, target, scope()).await,
15889 error_codes::SCOPE_NOT_CARRIER,
15890 "{opener:?} -> {target}"
15891 );
15892 }
15893 }
15894
15895 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15896 ScopeSelector::new(
15897 Principal::Reserved {
15898 module_id: OWNER.to_string(),
15899 },
15900 scope_ref.to_string(),
15901 scope_epoch,
15902 )
15903 }
15904
15905 #[tokio::test]
15906 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
15907 let mut rig = rig().await;
15908 rig.sync(vec![session(1)]).await;
15909 for opener in [OWNER, AFT] {
15910 assert_eq!(
15911 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
15912 .await,
15913 error_codes::SCOPE_EPOCH_REQUIRED,
15914 "{opener}"
15915 );
15916 }
15917 }
15918
15919 #[tokio::test]
15920 async fn admission_separates_not_synced_not_live_and_ended() {
15921 let mut rig = rig().await;
15922 let code = rig
15924 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15925 .await;
15926 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
15927 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
15928
15929 let ghost = ScopeSelector::new(
15931 Principal::Reserved {
15932 module_id: "ghost".to_string(),
15933 },
15934 "s".to_string(),
15935 Some(1),
15936 );
15937 assert_eq!(
15938 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
15939 error_codes::SCOPE_NOT_LIVE
15940 );
15941
15942 rig.sync(vec![session(2)]).await;
15943 assert_eq!(
15944 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
15945 .await,
15946 error_codes::SCOPE_NOT_LIVE
15947 );
15948 for epoch in [1, 3] {
15949 assert_eq!(
15950 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
15951 .await,
15952 error_codes::SCOPE_ENDED,
15953 "epoch {epoch}"
15954 );
15955 }
15956 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
15958 .await;
15959 }
15960
15961 #[tokio::test]
15962 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
15963 let mut rig = rig().await;
15964 rig.sync(vec![session(1)]).await;
15965 let route = rig
15966 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15967 .await;
15968 let stamp = route.stamp().expect("a scoped bind carries the stamp");
15969 assert_eq!(stamp.scope_ref, "s");
15970 assert_eq!(stamp.scope_epoch, 1);
15971 assert_eq!(stamp.kind, ScopeKind::Head);
15972 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
15973 assert!(stamp.attributes.delegates);
15974 assert!(stamp.owner_authorized);
15975 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15976 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
15977
15978 let (broca, mut broca_rx) = wide_ctx(50);
15981 hello_via_sink(
15982 &rig.handler,
15983 &broca,
15984 &mut broca_rx,
15985 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
15986 )
15987 .await;
15988 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
15989 .await
15990 .unwrap();
15991 let own = ScopeSelector::new(
15992 Principal::Reserved {
15993 module_id: BROCA.to_string(),
15994 },
15995 "b".to_string(),
15996 Some(1),
15997 );
15998 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
15999 assert!(!route.stamp().unwrap().owner_authorized);
16000 }
16001
16002 #[tokio::test]
16003 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
16004 let mut rig = rig().await;
16005 let mut record = head("s", 1);
16006 record.carriers = vec![carrier(AFT, None)];
16007 let flow_id = "Flow:run-7/step_2!~";
16008 record.attributes.flow_id = Some(flow_id.to_string());
16009 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
16010 .await
16011 .unwrap();
16012 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16013 panic!("not a sync reply");
16014 };
16015 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
16016 let route = rig
16017 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16018 .await;
16019 assert!(rig.live(&route), "the stamped bind committed");
16020 let stamp = route.stamp().expect("a flow scope carries a stamp");
16021 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
16022 assert_eq!(stamp.attributes.agent_id, None);
16023 assert!(!stamp.attributes.delegates);
16024 assert!(stamp.owner_authorized);
16025 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16026 assert_eq!(unscoped.stamp(), None);
16027 }
16028
16029 #[tokio::test]
16030 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
16031 let mut rig = rig_with_flow_support(false).await;
16032 let mut record = session(1);
16033 record.attributes.flow_id = Some("flow:7".to_string());
16034 rig.sync(vec![record]).await;
16035 for opener in [OWNER, AFT] {
16036 let body = rig
16037 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16038 .await;
16039 assert_eq!(body["code"], "target_flow_unsupported");
16040 let message = body["message"].as_str().unwrap();
16041 for required in [PLEXUS, "flow-scopes/v1"] {
16042 assert!(message.contains(required), "{message}");
16043 }
16044 }
16045 }
16046
16047 #[tokio::test]
16048 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
16049 let mut rig = rig_with_flow_support(true).await;
16050 let mut record = session(1);
16051 record.attributes.flow_id = Some("flow:7".to_string());
16052 rig.sync(vec![record]).await;
16053 for opener in [OWNER, AFT] {
16054 let route = rig
16055 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
16056 .await;
16057 assert!(rig.live(&route));
16058 assert_eq!(
16059 route.stamp().unwrap().attributes.flow_id.as_deref(),
16060 Some("flow:7")
16061 );
16062 }
16063 }
16064
16065 #[tokio::test]
16066 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
16067 use std::future::Future;
16068
16069 let mut rig = rig_with_flow_support(true).await;
16070 let mut record = session(1);
16071 record.attributes.flow_id = Some("flow:7".to_string());
16072 rig.sync(vec![record]).await;
16073 let (client, mut client_rx, frame) =
16074 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
16075 for _ in 0..64 {
16078 client.egress.try_send(route_bind_ack(1)).unwrap();
16079 }
16080 let handler = rig.handler.clone();
16081 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16082 std::future::poll_fn(|cx| {
16083 assert!(open.as_mut().poll(cx).is_pending());
16084 std::task::Poll::Ready(())
16085 })
16086 .await;
16087 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16088
16089 let old_connection = rig.modules[PLEXUS].0.connection_id;
16090 rig.handler.cleanup_connection(old_connection).unwrap();
16091 let (replacement, mut replacement_rx) = wide_ctx(200);
16092 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
16093 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
16094 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
16095 let hello = Frame::build(
16096 FrameType::Hello,
16097 control_flags(),
16098 0,
16099 0,
16100 200,
16101 serde_json::to_vec(&body).unwrap(),
16102 )
16103 .unwrap();
16104 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
16105
16106 client_rx.try_recv().unwrap();
16107 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16108 .await
16109 .expect("the replacement is refused without waiting for a bind ack")
16110 .unwrap();
16111 assert_eq!(replies.len(), 1);
16112 let body = parse_error(&replies[0]);
16113 assert_eq!(body["code"], "target_flow_unsupported");
16114 for required in [PLEXUS, "flow-scopes/v1"] {
16115 assert!(body["message"].as_str().unwrap().contains(required));
16116 }
16117 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
16118 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16119 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16120 assert!(rig
16121 .handler
16122 .registry
16123 .get_module_by_connection(replacement.connection_id)
16124 .unwrap()
16125 .is_some());
16126 }
16127
16128 #[tokio::test]
16129 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
16130 use std::future::Future;
16131 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16132 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16133 const NEXT_HASH: &str =
16134 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16135 let mut rig = rig().await;
16136 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
16137 record.carriers.push(ScopeCarrier::new(Principal::Direct));
16138 rig.sync(vec![record.clone()]).await;
16139 let (client, mut rx, frame) = rig.open_frame(
16140 None,
16141 PLEXUS,
16142 Some(
16143 rig.selector("s", Some(1))
16144 .with_bind_proof(Some(PROOF.into())),
16145 ),
16146 );
16147 for _ in 0..64 {
16150 client.egress.try_send(route_bind_ack(1)).unwrap();
16151 }
16152 let handler = rig.handler.clone();
16153 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16154 std::future::poll_fn(|cx| {
16155 assert!(open.as_mut().poll(cx).is_pending());
16156 std::task::Poll::Ready(())
16157 })
16158 .await;
16159 record.bind_proof_sha256 = Some(NEXT_HASH.into());
16160 rig.sync(vec![record]).await;
16161 rx.try_recv().unwrap();
16162 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16163 .await
16164 .unwrap()
16165 .unwrap();
16166 assert_eq!(
16167 parse_error(&replies[0])["code"],
16168 error_codes::SCOPE_BIND_PROOF_MISMATCH
16169 );
16170 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
16171 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16172 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16173 }
16174
16175 #[tokio::test]
16176 async fn scope_bind_proof_reserved_openers_ignore_proof() {
16177 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16178 let mut rig = rig().await;
16179 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16180 .await;
16181 for opener in [OWNER, AFT] {
16182 for proof in [None, Some("malformed+proof=")] {
16183 let route = rig
16184 .bound(
16185 Some(opener),
16186 PLEXUS,
16187 Some(
16188 rig.selector("s", Some(1))
16189 .with_bind_proof(proof.map(str::to_string)),
16190 ),
16191 )
16192 .await;
16193 assert!(rig.live(&route));
16194 }
16195 }
16196 }
16197
16198 #[tokio::test]
16199 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
16200 use std::future::Future;
16201 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16202 const NEXT_HASH: &str =
16203 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16204 let mut rig = rig().await;
16205 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16206 .await;
16207 let (client, mut rx, frame) =
16208 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
16209 for _ in 0..64 {
16210 client.egress.try_send(route_bind_ack(1)).unwrap();
16211 }
16212 let handler = rig.handler.clone();
16213 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16214 std::future::poll_fn(|cx| {
16215 assert!(open.as_mut().poll(cx).is_pending());
16216 std::task::Poll::Ready(())
16217 })
16218 .await;
16219 rig.sync(vec![
16220 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
16221 ])
16222 .await;
16223 rx.try_recv().unwrap();
16224 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16225 let bind = std::future::poll_fn(|cx| {
16226 assert!(open.as_mut().poll(cx).is_pending());
16227 module_rx.poll_recv(cx)
16228 })
16229 .await
16230 .unwrap()
16231 .frame;
16232 handler
16233 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
16234 .await
16235 .unwrap();
16236 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16237 .await
16238 .unwrap()
16239 .unwrap();
16240 assert_eq!(replies.len(), 1);
16241 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
16244 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16245 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
16246 let route = rig
16247 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
16248 .await;
16249 assert!(rig.live(&route));
16250 }
16251
16252 #[tokio::test]
16253 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
16254 {
16255 let mut rig = rig_with_flow_support(false).await;
16256 rig.sync(vec![session(1)]).await;
16257 let route = rig
16258 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16259 .await;
16260 assert!(rig.live(&route));
16261 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
16262 let mut record = session(1);
16263 record.attributes.flow_id = Some("flow:7".to_string());
16264 rig.sync(vec![record]).await;
16265 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
16266 assert!(rig.live(&unscoped));
16267 assert_eq!(unscoped.stamp(), None);
16268 }
16269
16270 #[tokio::test]
16271 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
16272 let mut rig = rig().await;
16273 let mut record = session(1);
16274 record.attributes.flow_id = Some("flow:7".to_string());
16275 rig.sync(vec![record.clone()]).await;
16276 let before = rig.forwarding.published_scope_tag(OWNER, "s");
16277 let mut owner_route = rig
16278 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16279 .await;
16280 let mut carrier_route = rig
16281 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
16282 .await;
16283 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16284 rig.sync(vec![record.clone()]).await;
16285 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16286 assert!(rig.live(&owner_route) && owner_route.untouched());
16287 assert!(rig.live(&carrier_route) && carrier_route.untouched());
16288
16289 record.attributes.flow_id = Some("flow:8".to_string());
16290 rig.sync(vec![record]).await;
16291 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
16292 let before = before.unwrap();
16293 assert_eq!(after.scope_epoch, before.scope_epoch);
16294 assert!(after.version > before.version);
16295 for route in [&mut owner_route, &mut carrier_route] {
16296 assert!(!rig.live(route));
16297 assert_eq!(
16298 route.closed_reason(),
16299 RouteCloseReason::ScopeDelegationChanged
16300 );
16301 }
16302 assert!(rig.live(&unscoped) && unscoped.untouched());
16303 for target in [PLEXUS, OTHER] {
16306 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
16307 let goodbye = module_rx
16308 .try_recv()
16309 .expect("the provider sees the drain")
16310 .frame;
16311 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16312 assert!(module_rx.try_recv().is_err());
16313 }
16314 let rebound = rig
16315 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16316 .await;
16317 assert_eq!(
16318 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
16319 Some("flow:8")
16320 );
16321 }
16322
16323 #[tokio::test]
16328 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
16329 let mut rig = rig().await;
16330 rig.sync(vec![session(1)]).await;
16331 let mut cotenant = rig
16332 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16333 .await;
16334
16335 let mut changed = session(1);
16336 changed.child_owners.push(Principal::Reserved {
16337 module_id: MAGIC.to_string(),
16338 });
16339 let mut ended = None;
16340 for (code, next) in [
16341 (error_codes::SCOPE_CHANGED, vec![changed]),
16342 (error_codes::SCOPE_ENDED, Vec::new()),
16343 ] {
16344 let relayed = rig
16345 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16346 .await;
16347 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
16348 ended = Some(next.is_empty());
16349 rig.sync(next).await;
16350 rig.ack(&relayed).await;
16351 let replies = relayed.task.await.unwrap();
16352 assert_eq!(replies.len(), 1, "{replies:?}");
16353 assert_eq!(parse_error(&replies[0])["code"], code);
16354 assert_eq!(
16355 subc_protocol::error_codes::is_retryable_route_open(code),
16356 code == error_codes::SCOPE_CHANGED
16357 );
16358 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16359 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16361 let mut goodbyes = Vec::new();
16364 while let Ok(outbound) = plexus_rx.try_recv() {
16365 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
16366 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
16367 }
16368 assert!(
16369 goodbyes.contains(&(bind_channel, bind_epoch)),
16370 "{goodbyes:?}"
16371 );
16372 assert!(rig
16373 .handler
16374 .registry
16375 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
16376 .unwrap()
16377 .is_some());
16378 }
16379 assert_eq!(ended, Some(true));
16380 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
16383 }
16384
16385 #[tokio::test]
16388 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
16389 struct Case {
16390 name: &'static str,
16391 change: fn(&mut ScopeRecord),
16392 closed: [Option<RouteCloseReason>; 4],
16395 }
16396 use RouteCloseReason::*;
16397 let cases = [
16398 Case {
16399 name: "a carrier entry removed",
16400 change: |r| {
16401 r.carriers.retain(|c| {
16402 c.principal
16403 != Principal::Reserved {
16404 module_id: AFT.to_string(),
16405 }
16406 })
16407 },
16408 closed: [None, Some(ScopeCarrierRemoved), None, None],
16409 },
16410 Case {
16411 name: "a target removed from a carrier",
16412 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
16413 closed: [None, None, None, Some(ScopeCarrierRemoved)],
16414 },
16415 Case {
16416 name: "a bare carrier narrowed to targets",
16417 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
16418 closed: [None, Some(ScopeCarrierRemoved), None, None],
16419 },
16420 Case {
16421 name: "delegates turned off",
16422 change: |r| r.attributes.delegates = false,
16423 closed: [Some(ScopeDelegationChanged); 4],
16424 },
16425 Case {
16426 name: "agent_id changed",
16427 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
16428 closed: [Some(ScopeDelegationChanged); 4],
16429 },
16430 Case {
16431 name: "a carrier added, child owners changed, the record re-sent",
16432 change: |r| {
16433 r.carriers.push(carrier(MAGIC, None));
16434 r.child_owners.push(Principal::Reserved {
16435 module_id: MAGIC.to_string(),
16436 });
16437 },
16438 closed: [None; 4],
16439 },
16440 Case {
16441 name: "a target added",
16442 change: |r| {
16443 r.carriers[1]
16444 .targets
16445 .as_mut()
16446 .unwrap()
16447 .push("third".to_string())
16448 },
16449 closed: [None; 4],
16450 },
16451 Case {
16452 name: "delegates turned on",
16453 change: |r| r.attributes.delegates = true,
16454 closed: [None; 4],
16455 },
16456 ];
16457 for case in cases {
16458 let mut rig = rig().await;
16459 rig.sync(vec![session(1)]).await;
16460 let scope = || Some(rig_selector("s", Some(1)));
16461 let mut routes = [
16462 rig.bound(Some(OWNER), PLEXUS, scope()).await,
16463 rig.bound(Some(AFT), PLEXUS, scope()).await,
16464 rig.bound(Some(BROCA), PLEXUS, scope()).await,
16465 rig.bound(Some(BROCA), OTHER, scope()).await,
16466 ];
16467 let mut record = session(1);
16468 (case.change)(&mut record);
16469 rig.sync(vec![record]).await;
16470 for (index, expected) in case.closed.iter().enumerate() {
16471 let route = &mut routes[index];
16472 match expected {
16473 Some(reason) => {
16474 assert!(!rig.live(route), "{}: route {index} still live", case.name);
16475 assert_eq!(
16476 route.closed_reason(),
16477 *reason,
16478 "{}: route {index}",
16479 case.name
16480 );
16481 }
16482 None => {
16483 assert!(rig.live(route), "{}: route {index} closed", case.name);
16484 assert!(
16485 route.untouched(),
16486 "{}: route {index} was told something",
16487 case.name
16488 );
16489 }
16490 }
16491 }
16492 }
16493 }
16494
16495 #[tokio::test]
16496 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
16497 for next in [Vec::new(), vec![session(2)]] {
16499 let mut rig = rig().await;
16500 rig.sync(vec![session(1)]).await;
16501 let mut route = rig
16502 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16503 .await;
16504 rig.sync(next).await;
16505 assert!(!rig.live(&route));
16506 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
16507 }
16508
16509 let mut rig = rig().await;
16512 let mut child = session(1);
16513 child.scope_ref = "child".to_string();
16514 child.kind = ScopeKind::Worker;
16515 child.parent = Some(ScopeParent::new(
16516 Principal::Reserved {
16517 module_id: OWNER.to_string(),
16518 },
16519 "s".to_string(),
16520 1,
16521 ));
16522 rig.sync(vec![session(1), child.clone()]).await;
16523 let mut child_route = rig
16524 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
16525 .await;
16526 assert_eq!(
16527 child_route.stamp().unwrap().parent_state,
16528 Some(ParentState::Linked)
16529 );
16530 rig.sync(vec![child]).await;
16531 assert!(!rig.live(&child_route));
16532 assert_eq!(
16533 child_route.closed_reason(),
16534 RouteCloseReason::ScopeParentEnded
16535 );
16536 }
16537
16538 #[tokio::test]
16539 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
16540 ) {
16541 let mut rig = rig().await;
16542 rig.sync(vec![session(1)]).await;
16543 let mut route = rig
16544 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16545 .await;
16546 let before = rig.forwarding.published_scope_tag(OWNER, "s");
16547 rig.sync(vec![session(1)]).await;
16548 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16549 assert!(rig.live(&route) && route.untouched());
16550
16551 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
16555 .forwarding
16556 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16557 .unwrap()
16558 else {
16559 panic!("the route is bound");
16560 };
16561 binding.flow.acquire_tagged(9, false).await.unwrap();
16562 let mut widened = session(1);
16563 widened.carriers.push(carrier(MAGIC, None));
16564 rig.sync(vec![widened]).await;
16565 assert!(rig.live(&route) && route.untouched());
16566 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16567 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
16568 assert_eq!(binding.flow.in_flight(), 1);
16571 binding
16572 .flow
16573 .acquire_tagged(10, false)
16574 .await
16575 .expect("the flow is still open");
16576 }
16577
16578 #[tokio::test]
16581 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
16582 let mut rig = rig().await;
16583 rig.sync(vec![session(1)]).await;
16584 let mut on_incumbent = rig
16585 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16586 .await;
16587
16588 let (candidate, _candidate_rx) = wide_ctx(9);
16591 let registration = rig
16592 .handler
16593 .registry
16594 .register_candidate_with_control_ops(
16595 manifest(PLEXUS, PROTOCOL_VERSION),
16596 PROTOCOL_VERSION,
16597 candidate.connection_id,
16598 module_baseline_control_ops(),
16599 )
16600 .unwrap();
16601 rig.forwarding
16602 .register_candidate_module_connection(
16603 candidate.connection_id,
16604 PLEXUS.to_string(),
16605 PROTOCOL_VERSION,
16606 manifest_concurrency(®istration.manifest),
16607 candidate.egress.clone(),
16608 )
16609 .unwrap();
16610 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
16611 rig.handler
16612 .registry
16613 .promote_candidate(PLEXUS)
16614 .unwrap()
16615 .unwrap();
16616 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
16617
16618 rig.sync(Vec::new()).await;
16619 assert!(!rig.live(&on_incumbent));
16620 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
16621 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16622 let goodbye = incumbent_rx
16623 .try_recv()
16624 .expect("the superseded endpoint is told")
16625 .frame;
16626 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16627 }
16628 }
16629}
16630
16631#[cfg(test)]
16632mod concurrency_default_exposure_tests {
16633 use super::*;
16634
16635 fn hello_body(role_json: &str) -> Vec<u8> {
16636 format!(
16637 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
16638 )
16639 .into_bytes()
16640 }
16641
16642 fn manifest_from(body: &[u8]) -> ModuleManifest {
16643 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
16644 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
16645 .expect("manifest parses")
16646 }
16647
16648 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
16649
16650 #[test]
16651 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
16652 let body = hello_body(&format!(
16653 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
16654 ));
16655 let manifest = manifest_from(&body);
16656 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16659 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
16660 }
16661
16662 #[test]
16663 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
16664 let body = hello_body(&format!(
16665 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
16666 ));
16667 let manifest = manifest_from(&body);
16668 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16669 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16670 }
16671
16672 #[test]
16673 fn non_management_roles_are_never_reported() {
16674 let body = hello_body(
16675 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
16676 );
16677 let manifest = manifest_from(&body);
16678 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16679 }
16680}