1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
21 error_codes,
22 manifest::{
23 validate_hello_capability_grammar, validate_hello_event_declarations,
24 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
25 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
26 },
27 scope::{
28 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
29 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
30 SCOPE_SYNC_OP,
31 },
32 session::{
33 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
34 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
35 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
36 ROLE_VERSIONS_FIELD,
37 },
38 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
39 Priority, RouteTarget, PROTOCOL_VERSION,
40};
41use tokio::time::{timeout_at, Instant};
42use tracing::{debug, info, warn};
43
44use crate::{
45 capability_requirements::{
46 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
47 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
48 RuntimeModule,
49 },
50 daemon_config::RestartRequiredSection,
51 forwarding::{
52 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
53 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
54 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
55 },
56 observability::{
57 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
58 },
59 provenance::{
60 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
61 },
62 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
63 router::{RouteCtx, RouterError},
64 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
65 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
66 stderr_tail::{CaptureState, TailEntry},
67 supervise::{
68 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
69 SupervisorHandle, SwapHelloAdmission,
70 },
71 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
72};
73
74pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
80
81const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
82const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
83const CAP_PING_PONG: &str = "ping_pong_v1";
84const CAP_SESSION_ATTACH: &str = "session_attach_v1";
85const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
86
87const SUBC_CONTROL_OPS: &[&str] = &[
88 ops::ACTIVITY_RECORD,
89 ops::ACTIVITY_QUERY,
90 ops::ACTIVITY_SUBSCRIBE,
91 ops::SERVER_DESCRIBE,
92 ops::CATALOG_LIST,
93 ops::ROUTE_OPEN,
94 ops::ROUTE_POLL,
95 ops::ROUTE_CLOSING,
96 ops::ROUTE_CLOSED,
97 ops::SUPERVISOR_LIST,
98 ops::SUPERVISOR_RESTART,
99 ops::SUPERVISOR_SWAP,
100 ops::SUPERVISOR_RELOAD,
101 ops::SUPERVISOR_RESCAN,
102 ops::SUPERVISOR_RELEASE_RESERVED,
103 ops::SUPERVISOR_SET_ENABLED,
104 ops::SUPERVISOR_HEALTH_PROBE,
105 ops::SUPERVISOR_HEALTH,
106 ops::SUPERVISOR_STDERR_TAIL,
107 ops::SUPERVISOR_TERMINALS,
108 ops::SUPERVISOR_ROUTES,
109 ops::SUPERVISOR_PROVENANCE,
110 ops::SUPERVISOR_SPAWN_SNAPSHOT,
111 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
112];
113
114const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
115 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
116 "supervisor.live_roots",
117 SCOPE_SYNC_OP,
118 SCOPE_APPLY_OP,
119 SCOPE_DESCRIBE_OP,
120 "operator.confirm",
121];
122
123const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
127
128const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
129
130pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
138
139pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
154
155pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
173
174const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
175const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
176
177fn reload_verdict(
178 configured: &Path,
179 spawned_from: Option<&Path>,
180 image: subc_control::RunningImageAgreement,
181) -> PendingReloadVerdict {
182 let path = match spawned_from {
183 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
184 ReloadPathAgreement::Match
185 }
186 Some(spawned_from) => ReloadPathAgreement::Mismatch {
187 configured: configured.to_path_buf(),
188 spawned_from: spawned_from.to_path_buf(),
189 },
190 None => ReloadPathAgreement::Unavailable {
191 reason: if matches!(
192 image,
193 subc_control::RunningImageAgreement::Unavailable {
194 reason: subc_control::RunningImageUnavailableReason::NotRunning
195 }
196 ) {
197 ReloadPathUnavailableReason::NotRunning
198 } else {
199 ReloadPathUnavailableReason::SpawnedPathUnavailable
200 },
201 },
202 };
203 PendingReloadVerdict { path, image }
204}
205
206fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
207 if configured == spawned_from {
208 return true;
209 }
210 #[cfg(windows)]
211 {
212 match (
218 std::fs::canonicalize(configured),
219 std::fs::canonicalize(spawned_from),
220 ) {
221 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
222 _ => false,
223 }
224 }
225 #[cfg(not(windows))]
226 {
227 false
228 }
229}
230
231#[derive(Clone)]
232struct DaemonProvenanceFacts {
233 build: DaemonBuildProvenance,
234 pid: Option<u32>,
235 started_at_ms: Option<u64>,
236 start_clock: Option<crate::clock::StartClock>,
237 executable_path: Option<PathBuf>,
238 executable_identity: Option<SpawnedFileIdentity>,
239 process_start_time: Option<u64>,
240 probe: ExecutableIdentityProbe,
241}
242
243impl Default for DaemonProvenanceFacts {
244 fn default() -> Self {
245 Self {
246 build: DaemonBuildProvenance {
247 build_git_sha: None,
248 build_lock_digest: None,
249 },
250 pid: None,
251 started_at_ms: None,
252 start_clock: None,
253 executable_path: None,
254 executable_identity: None,
255 process_start_time: None,
256 probe: ExecutableIdentityProbe::default(),
257 }
258 }
259}
260
261#[derive(Debug, Clone)]
262struct SupervisorRescanContext {
263 supervisor: Supervisor,
264 config_path: PathBuf,
265 configured_port: Option<u16>,
266 storage_config: Option<crate::daemon_config::StorageConfig>,
267 admission_facts_carrier_module_id: Option<String>,
268 admission_facts_targets: Option<Vec<String>>,
269 scope_authority_owners: Vec<String>,
270}
271
272const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
293 "reloading",
294 "supervisor_not_live",
295 "registration_not_active",
296 "no_forwarding_connection",
297 "relay_send_failed",
298];
299
300#[derive(Clone)]
302pub struct ControlHandler {
303 registry: Arc<Registry>,
304 activity: Option<crate::activity::ActivityLedger>,
305 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
306 forwarding: Arc<ForwardingTable>,
307 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
308 supervisor: SupervisorHandle,
309 subc_capabilities: Arc<[String]>,
310 route_bind_relay_timeout: Duration,
314 route_bind_relay_timeouts: BTreeMap<String, Duration>,
318 route_bind_breakers: RouteBindBreakers,
321 route_bind_concurrency: RouteBindConcurrency,
324 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
329 route_bind_breaker_threshold: u32,
331 route_bind_breaker_cooldown: Duration,
333 health_probe_timeout: Duration,
334 storage_config: Option<crate::daemon_config::StorageConfig>,
337 machine_id: Option<crate::machine_id::MachineId>,
341 admission_facts_carrier_module_id: Option<String>,
342 admission_facts_targets: Option<Vec<String>>,
343 scopes: Arc<RwLock<ScopeTable>>,
347 scope_deadline_added: Arc<tokio::sync::Notify>,
351 scope_authority_owners: Vec<String>,
354 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
357 rescan: Option<SupervisorRescanContext>,
358 connected_clients: ConnectedClients,
359 counters: DaemonCounters,
360 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
361 daemon_provenance: DaemonProvenanceFacts,
362 #[cfg(test)]
363 control_dispatch_delay: Option<Duration>,
364 #[cfg(test)]
365 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
366 #[cfg(test)]
370 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
371}
372
373impl fmt::Debug for ControlHandler {
374 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
375 f.debug_struct("ControlHandler")
376 .field("registry", &self.registry)
377 .field("forwarding", &self.forwarding)
378 .field("process_liveness", &self.process_liveness.is_some())
379 .field("supervisor", &self.supervisor)
380 .field("subc_capabilities", &self.subc_capabilities)
381 .finish()
382 }
383}
384
385struct RouteOpenRequest {
386 target: RouteTarget,
387 identity: BindIdentity,
388 consumer_identity: Option<ConsumerIdentity>,
389 consumer_capabilities: Option<Vec<String>>,
390 role_versions: Option<BTreeMap<String, String>>,
391 admission_facts: Option<serde_json::Value>,
392 scope: Option<ScopeSelector>,
393}
394
395struct RouteBindReservationGuard {
396 forwarding: Arc<ForwardingTable>,
397 endpoint: ModuleEndpointId,
398 relay_corr: u64,
399 armed: bool,
400}
401
402struct ModuleControlRpcGuard {
403 forwarding: Arc<ForwardingTable>,
404 endpoint: ModuleEndpointId,
405 corr: u64,
406 armed: bool,
407}
408
409impl ModuleControlRpcGuard {
410 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
411 Self {
412 forwarding,
413 endpoint,
414 corr,
415 armed: true,
416 }
417 }
418
419 fn disarm(&mut self) {
420 self.armed = false;
421 }
422}
423
424impl Drop for ModuleControlRpcGuard {
425 fn drop(&mut self) {
426 if self.armed {
427 let _ = self
428 .forwarding
429 .cancel_module_control_rpc(self.endpoint, self.corr);
430 }
431 }
432}
433
434impl RouteBindReservationGuard {
435 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
436 Self {
437 forwarding,
438 endpoint,
439 relay_corr,
440 armed: true,
441 }
442 }
443
444 fn release_and_disarm(&mut self) {
445 if !self.armed {
446 return;
447 }
448 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
449 self.endpoint,
450 self.relay_corr,
451 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
452 ) {
453 send_goodbye_target_best_effort(
454 &self.forwarding.counters(),
455 &target,
456 "canceled route.bind",
457 );
458 }
459 self.armed = false;
460 }
461
462 fn disarm(&mut self) {
463 self.armed = false;
464 }
465}
466
467impl Drop for RouteBindReservationGuard {
468 fn drop(&mut self) {
469 self.release_and_disarm();
470 }
471}
472
473#[derive(Debug, Clone, Default)]
503pub(crate) struct RouteBindBreakers {
504 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
505}
506
507#[derive(Debug, Clone, Default)]
508pub(crate) struct RouteBindConcurrency {
509 modules: Arc<Mutex<HashMap<String, usize>>>,
510}
511
512struct RouteBindConcurrencyGuard {
513 concurrency: RouteBindConcurrency,
514 module_id: String,
515}
516
517impl RouteBindConcurrency {
518 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
521 let mut modules = self
522 .modules
523 .lock()
524 .expect("route.bind concurrency mutex poisoned");
525 let in_flight = modules.entry(module_id.to_string()).or_default();
526 if *in_flight >= limit {
527 return Err(*in_flight);
528 }
529 *in_flight += 1;
530 Ok(RouteBindConcurrencyGuard {
531 concurrency: self.clone(),
532 module_id: module_id.to_string(),
533 })
534 }
535}
536
537impl Drop for RouteBindConcurrencyGuard {
538 fn drop(&mut self) {
539 let mut modules = self
540 .concurrency
541 .modules
542 .lock()
543 .expect("route.bind concurrency mutex poisoned");
544 let remove = {
545 let in_flight = modules
546 .get_mut(&self.module_id)
547 .expect("admitted route.bind has a concurrency entry");
548 *in_flight -= 1;
549 *in_flight == 0
550 };
551 if remove {
552 modules.remove(&self.module_id);
553 }
554 }
555}
556
557#[derive(Debug, Default)]
558struct ModuleBreakerState {
559 consecutive_timeouts: u32,
561 cooldown_until: Option<Instant>,
564 probe_in_flight: Option<Arc<()>>,
569}
570
571enum RouteBindAdmission<'a> {
573 Admitted {
574 guard: RouteBindBreakerGuard<'a>,
575 probe: bool,
578 },
579 Refused {
580 consecutive_timeouts: u32,
581 retry_in: Duration,
585 probe_in_flight: bool,
586 },
587}
588
589struct RouteBindBreakerGuard<'a> {
596 breakers: RouteBindBreakers,
597 module_id: &'a str,
598 probe_token: Option<Arc<()>>,
599 settled: bool,
600}
601
602impl RouteBindBreakerGuard<'_> {
603 fn record_accepted(&mut self) -> bool {
607 self.settled = true;
608 self.breakers.record_accepted(self.module_id)
609 }
610
611 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
614 self.settled = true;
615 self.breakers.record_timeout(
616 self.module_id,
617 self.probe_token.as_ref(),
618 threshold,
619 cooldown,
620 )
621 }
622
623 fn record_inconclusive(&mut self) {
632 self.settled = true;
633 self.breakers
634 .record_inconclusive(self.module_id, self.probe_token.as_ref());
635 }
636}
637
638impl Drop for RouteBindBreakerGuard<'_> {
639 fn drop(&mut self) {
640 if !self.settled {
641 self.breakers
642 .record_inconclusive(self.module_id, self.probe_token.as_ref());
643 }
644 }
645}
646
647struct BreakerOpened {
651 consecutive_timeouts: u32,
652 reopened_after_probe: bool,
655}
656
657impl RouteBindBreakers {
658 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
659 self.modules
660 .lock()
661 .expect("route.bind breaker mutex poisoned")
662 }
663
664 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
667 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
668 probe: probe_token.is_some(),
669 guard: RouteBindBreakerGuard {
670 breakers: self.clone(),
671 module_id,
672 probe_token,
673 settled: false,
674 },
675 };
676
677 let mut modules = self.lock();
678 let Some(state) = modules.get_mut(module_id) else {
679 return admitted(None);
680 };
681 let Some(cooldown_until) = state.cooldown_until else {
682 return admitted(None);
683 };
684 if state.probe_in_flight.is_some() {
685 return RouteBindAdmission::Refused {
686 consecutive_timeouts: state.consecutive_timeouts,
687 retry_in: Duration::ZERO,
688 probe_in_flight: true,
689 };
690 }
691 let now = Instant::now();
692 if now < cooldown_until {
693 return RouteBindAdmission::Refused {
694 consecutive_timeouts: state.consecutive_timeouts,
695 retry_in: cooldown_until - now,
696 probe_in_flight: false,
697 };
698 }
699 let token = Arc::new(());
700 state.probe_in_flight = Some(Arc::clone(&token));
701 admitted(Some(token))
702 }
703
704 fn record_accepted(&self, module_id: &str) -> bool {
705 self.lock()
706 .remove(module_id)
707 .is_some_and(|state| state.cooldown_until.is_some())
708 }
709
710 fn record_timeout(
711 &self,
712 module_id: &str,
713 probe_token: Option<&Arc<()>>,
714 threshold: u32,
715 cooldown: Duration,
716 ) -> Option<BreakerOpened> {
717 let mut modules = self.lock();
718 let state = modules.entry(module_id.to_string()).or_default();
719 let was_open = state.cooldown_until.is_some();
720 let was_probe = Self::owns_probe(state, probe_token);
721 if was_probe {
722 state.probe_in_flight = None;
723 }
724 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
725 if state.consecutive_timeouts < threshold {
726 return None;
727 }
728 state.cooldown_until = Some(Instant::now() + cooldown);
729 Some(BreakerOpened {
730 consecutive_timeouts: state.consecutive_timeouts,
731 reopened_after_probe: was_open && was_probe,
732 })
733 }
734
735 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
736 state
740 .probe_in_flight
741 .as_ref()
742 .zip(token)
743 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
744 }
745
746 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
747 if let Some(state) = self.lock().get_mut(module_id) {
748 if Self::owns_probe(state, probe_token) {
749 state.probe_in_flight = None;
750 }
751 }
752 }
753
754 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
769 self.lock()
770 .remove(module_id)
771 .map(|state| state.consecutive_timeouts)
772 .filter(|discarded| *discarded > 0)
773 }
774
775 fn open_snapshot(&self) -> Option<serde_json::Value> {
782 let now = Instant::now();
783 let modules = self.lock();
784 let open = modules
785 .iter()
786 .filter_map(|(module_id, state)| {
787 let cooldown_until = state.cooldown_until?;
788 Some((
789 module_id.clone(),
790 serde_json::json!({
791 "consecutive_timeouts": state.consecutive_timeouts,
792 "cooldown_remaining_ms":
793 cooldown_until.saturating_duration_since(now).as_millis() as u64,
794 "probe_in_flight": state.probe_in_flight.is_some(),
795 }),
796 ))
797 })
798 .collect::<serde_json::Map<String, serde_json::Value>>();
799 (!open.is_empty()).then_some(serde_json::Value::Object(open))
800 }
801}
802
803impl ControlHandler {
804 pub fn new(registry: Arc<Registry>) -> Self {
805 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
806 }
807
808 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
809 let counters = forwarding.counters();
810 let route_bind_breakers = forwarding.route_bind_breakers();
814 let route_bind_concurrency = forwarding.route_bind_concurrency();
815 let route_outages = forwarding.route_outages();
816 Self {
817 registry,
818 activity: None,
819 activity_streams: Arc::new(Mutex::new(HashMap::new())),
820 forwarding,
821 process_liveness: None,
822 supervisor: SupervisorHandle::new(),
823 subc_capabilities: Arc::from([
824 CAP_MANIFEST_REGISTRATION.to_string(),
825 CAP_CHANNEL_LIFECYCLE.to_string(),
826 CAP_PING_PONG.to_string(),
827 CAP_SESSION_ATTACH.to_string(),
828 CAP_ADMISSION_FACTS_RELAY.to_string(),
829 CAP_SCOPES_V1.to_string(),
830 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
831 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
832 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
833 ]),
834 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
835 route_bind_relay_timeouts: BTreeMap::new(),
836 route_bind_breakers,
837 route_bind_concurrency,
838 route_outages,
839 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
840 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
841 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
842 storage_config: None,
843 machine_id: None,
844 admission_facts_carrier_module_id: None,
845 admission_facts_targets: None,
846 scopes: Arc::new(RwLock::new(ScopeTable::new(
847 crate::daemon_config::default_scope_authority_owners(),
848 ))),
849 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
850 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
851 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
852 rescan: None,
853 connected_clients: ConnectedClients::new(),
854 counters,
855 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
856 daemon_provenance: DaemonProvenanceFacts::default(),
857 #[cfg(test)]
858 control_dispatch_delay: None,
859 #[cfg(test)]
860 provenance_probe_override: None,
861 #[cfg(test)]
862 route_bind_relay_sent_hold: None,
863 }
864 }
865
866 pub fn with_storage_config(
869 mut self,
870 storage_config: Option<crate::daemon_config::StorageConfig>,
871 ) -> Self {
872 self.storage_config = storage_config;
873 self
874 }
875
876 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
879 self.activity = Some(crate::activity::ActivityLedger::open(
880 run_dir.join("activity.jsonl"),
881 incarnation,
882 ));
883 self.supervisor.set_activity(self.activity.clone());
884 self
885 }
886
887 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
890 self.machine_id = machine_id;
891 self
892 }
893
894 pub fn with_admission_facts_config(
898 mut self,
899 carrier_module_id: Option<String>,
900 targets: Option<Vec<String>>,
901 ) -> Self {
902 self.admission_facts_carrier_module_id = carrier_module_id;
903 self.admission_facts_targets = targets;
904 self
905 }
906
907 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
911 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
912 self.scope_authority_owners = owners;
913 self
914 }
915
916 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
919 self.route_bind_relay_timeout = timeout;
920 self
921 }
922
923 pub fn with_route_bind_relay_timeouts(
929 mut self,
930 timeouts: impl IntoIterator<Item = (String, Duration)>,
931 ) -> Self {
932 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
933 self
934 }
935
936 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
942 self.route_bind_relay_timeouts
943 .get(module_id)
944 .copied()
945 .unwrap_or(self.route_bind_relay_timeout)
946 }
947
948 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
956 self.route_bind_breaker_threshold = threshold.max(1);
957 self.route_bind_breaker_cooldown = cooldown;
958 self
959 }
960
961 #[cfg(test)]
962 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
963 self.health_probe_timeout = timeout;
964 self
965 }
966
967 #[cfg(test)]
968 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
969 self.control_dispatch_delay = Some(delay);
970 self
971 }
972
973 #[cfg(test)]
977 pub(crate) fn with_route_bind_relay_sent_hold(
978 mut self,
979 hold: Arc<tokio::sync::Barrier>,
980 ) -> Self {
981 self.route_bind_relay_sent_hold = Some(hold);
982 self
983 }
984
985 pub fn with_process_liveness(
986 mut self,
987 process_liveness: Arc<dyn ModuleProcessLiveness>,
988 ) -> Self {
989 self.process_liveness = Some(process_liveness);
990 self
991 }
992
993 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
994 self.supervisor = supervisor;
995 self.supervisor.set_activity(self.activity.clone());
996 self
997 }
998
999 pub fn with_daemon_provenance(
1000 mut self,
1001 pid: u32,
1002 started_at_ms: u64,
1003 executable_path: Option<PathBuf>,
1004 build_git_sha: Option<String>,
1005 build_lock_digest: Option<String>,
1006 ) -> Self {
1007 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1008 let process_start_time = process_start_time(pid);
1009 self.daemon_provenance = DaemonProvenanceFacts {
1010 build: DaemonBuildProvenance {
1011 build_git_sha,
1012 build_lock_digest,
1013 },
1014 pid: Some(pid),
1015 started_at_ms: Some(started_at_ms),
1016 start_clock: None,
1017 executable_path,
1018 executable_identity,
1019 process_start_time,
1020 probe: ExecutableIdentityProbe::default(),
1021 };
1022 self
1023 }
1024
1025 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1026 self.daemon_provenance.start_clock = Some(clock);
1027 self
1028 }
1029
1030 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1032 self.scopes
1033 .write()
1034 .unwrap_or_else(|poisoned| poisoned.into_inner())
1035 .set_wall_clock(Arc::new(clock));
1036 self
1037 }
1038
1039 #[cfg(test)]
1040 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1041 self.provenance_probe_override = Some(result);
1042 self
1043 }
1044
1045 pub fn with_capability_config(
1049 self,
1050 modules: impl IntoIterator<Item = (String, bool)>,
1051 reserved_capabilities: BTreeMap<String, String>,
1052 ) -> Self {
1053 self.capability_evaluator
1054 .configure(modules, reserved_capabilities);
1055 self
1056 }
1057
1058 pub fn with_supervisor_rescan(
1059 mut self,
1060 supervisor: Supervisor,
1061 config_path: impl Into<PathBuf>,
1062 configured_port: Option<u16>,
1063 ) -> Self {
1064 self.rescan = Some(SupervisorRescanContext {
1065 supervisor,
1066 config_path: config_path.into(),
1067 configured_port,
1068 storage_config: self.storage_config.clone(),
1069 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1070 admission_facts_targets: self.admission_facts_targets.clone(),
1071 scope_authority_owners: self.scope_authority_owners.clone(),
1072 });
1073 self
1074 }
1075
1076 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1077 self.connected_clients = connected_clients;
1078 self
1079 }
1080
1081 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1082 Arc::clone(&self.forwarding)
1083 }
1084
1085 pub(crate) fn counters(&self) -> DaemonCounters {
1086 self.counters.clone()
1087 }
1088
1089 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1092 tokio::spawn(async move {
1093 loop {
1094 self.capability_evaluator
1095 .wait_for_change_or_deadline()
1096 .await;
1097 self.refresh_capability_requirements();
1098 }
1099 });
1100 }
1101
1102 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1108 tokio::spawn(async move {
1109 loop {
1110 let has_deadlines = self
1111 .scopes
1112 .read()
1113 .unwrap_or_else(|poisoned| poisoned.into_inner())
1114 .has_deadlines();
1115 if !has_deadlines {
1116 self.scope_deadline_added.notified().await;
1117 continue;
1118 }
1119 tokio::time::sleep(Duration::from_secs(1)).await;
1120 if let Err(error) = self.sweep_expired_scopes() {
1121 warn!(%error, "scope expiry sweep failed");
1122 }
1123 }
1124 });
1125 }
1126
1127 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1128 let mut table = self
1129 .scopes
1130 .write()
1131 .unwrap_or_else(|poisoned| poisoned.into_inner());
1132 let swept = table.sweep_expired();
1133 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1134 drop(table);
1135 self.close_scope_drained_routes(drained);
1136 Ok(())
1137 }
1138
1139 fn publish_scope_changes(
1140 &self,
1141 changes: &[crate::scopes::ScopeTagChange],
1142 expired: &[crate::scopes::ScopeExpired],
1143 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1144 if changes.is_empty() {
1145 return Ok(Vec::new());
1146 }
1147 let drained = self
1148 .forwarding
1149 .publish_scope_changes(changes)
1150 .map_err(RouterError::Forwarding)?;
1151 let mut counts = HashMap::new();
1152 for route in &drained {
1153 *counts
1154 .entry((
1155 route.scope.owner.as_str(),
1156 route.scope.scope_ref.as_str(),
1157 route.scope.tag.scope_epoch,
1158 ))
1159 .or_insert(0usize) += 1;
1160 }
1161 for scope in expired {
1162 let routes_closed = counts
1163 .get(&(
1164 scope.owner.as_str(),
1165 scope.scope_ref.as_str(),
1166 scope.scope_epoch,
1167 ))
1168 .copied()
1169 .unwrap_or(0);
1170 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1171 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1172 routes_closed, "scope expired");
1173 }
1174 Ok(drained)
1175 }
1176
1177 fn runtime_capability_snapshot(
1178 &self,
1179 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1180 let runtime = self
1181 .supervisor
1182 .list()
1183 .into_iter()
1184 .map(|module| {
1185 let status = module.status().map_err(|err| {
1186 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1187 })?;
1188 Ok(RuntimeModule {
1189 module_id: status.module_id,
1190 state: status.state,
1191 enabled: status.enabled,
1192 })
1193 })
1194 .collect::<Result<Vec<_>, RouterError>>()?;
1195 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1196 RouterError::backend(
1197 0,
1198 0,
1199 format!("failed to list capability registrations: {err}"),
1200 )
1201 })?;
1202 let registrations = registrations
1203 .into_iter()
1204 .map(|registration| RegisteredModule {
1205 module_id: registration.manifest.module_id,
1206 module_version: registration.manifest.module_version,
1207 capabilities: registration.manifest.capabilities,
1208 })
1209 .collect();
1210 Ok((runtime, registrations))
1211 }
1212
1213 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1220 let cached_registration = RegisteredModule {
1221 module_id: registration.manifest.module_id.clone(),
1222 module_version: registration.manifest.module_version.clone(),
1223 capabilities: registration.manifest.capabilities.clone(),
1224 };
1225 if self.capability_evaluator.record_hello(&cached_registration) {
1226 warn!(
1227 module_id = %cached_registration.module_id,
1228 "capability claims drifted from the cached manifest"
1229 );
1230 }
1231 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1232 self.enforce_capability_denies();
1233 }
1234 self.refresh_capability_requirements();
1235 }
1236
1237 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1241 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1242 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1243 self.supervisor.set_swap_promotion_observer(observer);
1244 }
1245
1246 pub fn refresh_capability_requirements(&self) {
1247 match self.runtime_capability_snapshot() {
1248 Ok((runtime, registrations)) => {
1249 log_requirement_events(
1250 self.capability_evaluator
1251 .evaluate_now(&runtime, ®istrations),
1252 );
1253 }
1254 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1255 }
1256 }
1257
1258 fn enforce_capability_denies(&self) {
1262 let (_, registrations) = match self.registry.list_modules() {
1263 Ok(snapshot) => snapshot,
1264 Err(err) => {
1265 warn!(error = %err, "failed to read registrations for capability deny census");
1266 return;
1267 }
1268 };
1269 let manifests = registrations
1270 .into_iter()
1271 .map(|registration| {
1272 (
1273 registration.manifest.module_id.clone(),
1274 registration.manifest,
1275 )
1276 })
1277 .collect::<BTreeMap<_, _>>();
1278 let census = match self.forwarding.route_census(None) {
1279 Ok(census) => census,
1280 Err(err) => {
1281 warn!(error = %err, "failed to read route census for capability deny enforcement");
1282 return;
1283 }
1284 };
1285
1286 for (target_module_id, routes) in census {
1287 let Some(target_manifest) = manifests.get(&target_module_id) else {
1288 continue;
1289 };
1290 let mut closed_routes = Vec::new();
1291 let mut module_goodbyes = Vec::new();
1292 for route in routes {
1293 let Principal::Reserved {
1294 module_id: opening_module_id,
1295 } = &route.principal
1296 else {
1297 continue;
1298 };
1299 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1300 continue;
1301 };
1302 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1303 continue;
1304 };
1305
1306 match self.forwarding.release_client_route(
1307 route.goodbye_target.connection_id,
1308 route.goodbye_target.channel,
1309 route.goodbye_target.epoch,
1310 ) {
1311 Ok(RouteRelease::Removed(module_goodbye)) => {
1312 warn!(
1313 opening_module_id,
1314 target_module_id,
1315 capability,
1316 "force-closing route because an attested capability deny edge now matches"
1317 );
1318 closed_routes.push(route);
1319 module_goodbyes.push(module_goodbye);
1320 }
1321 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1322 Err(err) => warn!(
1323 opening_module_id,
1324 target_module_id,
1325 capability,
1326 error = %err,
1327 "failed to force-close capability-denied route"
1328 ),
1329 }
1330 }
1331
1332 if closed_routes.is_empty() {
1333 continue;
1334 }
1335 send_route_control_pushes(
1336 &self.forwarding,
1337 closed_routes,
1338 ClientControlPush::RouteClosed {
1339 module_id: target_module_id,
1340 channels: Vec::new(),
1341 reason: RouteCloseReason::CapabilityDenied,
1342 drained: false,
1343 abandoned: 0,
1344 excluded_subscriptions: 0,
1345 terminal: Some(false),
1346 },
1347 );
1348 self.emit_route_goodbyes(module_goodbyes);
1349 }
1350 }
1351
1352 fn not_ready_reason(
1358 &self,
1359 registration: &crate::registry::ModuleRegistration,
1360 ) -> Option<NotReadyReason> {
1361 if !registration.ready {
1362 return Some(NotReadyReason {
1363 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1364 capability: None,
1365 });
1366 }
1367 self.first_unprovided_required_capability(registration)
1368 .map(|capability| NotReadyReason {
1369 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1370 capability: Some(capability),
1371 })
1372 }
1373
1374 fn first_unprovided_required_capability(
1392 &self,
1393 registration: &crate::registry::ModuleRegistration,
1394 ) -> Option<String> {
1395 let required = registration
1396 .manifest
1397 .capabilities
1398 .iter()
1399 .flat_map(|declarations| declarations.requires.iter())
1400 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1401 .map(|requirement| requirement.capability.as_str())
1402 .collect::<BTreeSet<_>>();
1403 if required.is_empty() {
1404 return None;
1405 }
1406 let module_id = registration.manifest.module_id.as_str();
1407 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1408 if required
1409 .iter()
1410 .any(|capability| verdict(capability).is_none())
1411 {
1412 self.refresh_capability_requirements();
1413 }
1414 required
1415 .into_iter()
1416 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1417 .map(str::to_string)
1418 }
1419
1420 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1421 self.capability_evaluator
1422 .statuses()
1423 .into_iter()
1424 .map(capability_requirement_status)
1425 .collect()
1426 }
1427
1428 fn deregister_connection(
1435 &self,
1436 connection_id: ConnectionId,
1437 reason: RegistrationEndReason,
1438 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1439 self.registry
1440 .deregister_connection_with_reason(connection_id, reason)
1441 }
1442
1443 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1444 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1445 return None;
1446 }
1447 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1448 parse_client_control_request(&frame.body)
1449 else {
1450 return None;
1451 };
1452 Some(target_module_id(&target).to_string())
1453 }
1454
1455 pub(crate) fn route_open_capacity_refusal(
1456 &self,
1457 ctx: &RouteCtx,
1458 frame: &Frame,
1459 target_module_id: &str,
1460 in_flight: usize,
1461 limit: usize,
1462 ) -> Result<Frame, RouterError> {
1463 self.route_open_admission_refusal_frame(
1464 ctx,
1465 frame,
1466 target_module_id,
1467 "open_admission_full",
1468 (in_flight, limit),
1469 format!(
1470 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1471 ),
1472 )
1473 }
1474
1475 fn route_open_target_capacity_refusal(
1476 &self,
1477 ctx: &RouteCtx,
1478 frame: &Frame,
1479 target_module_id: &str,
1480 in_flight: usize,
1481 ) -> Result<Frame, RouterError> {
1482 self.route_open_admission_refusal_frame(
1483 ctx,
1484 frame,
1485 target_module_id,
1486 "target_binds_full",
1487 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1488 format!(
1489 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1490 ),
1491 )
1492 }
1493
1494 fn route_open_admission_refusal_frame(
1502 &self,
1503 ctx: &RouteCtx,
1504 frame: &Frame,
1505 target_module_id: &str,
1506 reason: &'static str,
1507 (in_flight, limit): (usize, usize),
1508 message: impl Into<String>,
1509 ) -> Result<Frame, RouterError> {
1510 let code = error_codes::TARGET_UNAVAILABLE;
1511 self.counters.increment_route_open_refused(code);
1512 info!(
1513 target: "control",
1514 code,
1515 reason,
1516 module_id = ?target_module_id,
1517 connection_id = ctx.connection_id.get(),
1518 in_flight,
1519 limit,
1520 "route.open refused"
1521 );
1522 control_error_frame(frame, code, message.into())
1523 }
1524
1525 #[cfg(test)]
1531 pub fn handle_control(
1532 &self,
1533 connection_id: ConnectionId,
1534 frame: Frame,
1535 ) -> Result<Vec<Frame>, RouterError> {
1536 match frame.header.ty {
1537 FrameType::Ping => Ok(vec![pong(&frame)?]),
1538 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1539 FrameType::Goodbye => self.handle_goodbye(connection_id),
1540 ty => Ok(vec![control_error_frame(
1541 &frame,
1542 "unsupported_control_frame",
1543 format!("unsupported channel-0 frame {ty:?}"),
1544 )?]),
1545 }
1546 }
1547
1548 pub async fn handle_control_frame(
1549 &self,
1550 ctx: &RouteCtx,
1551 frame: Frame,
1552 ) -> Result<Vec<Frame>, RouterError> {
1553 self.handle_control_frame_timed(ctx, frame, None).await
1554 }
1555
1556 pub(crate) async fn handle_control_frame_timed(
1557 &self,
1558 ctx: &RouteCtx,
1559 frame: Frame,
1560 dispatch_started_at: Option<StdInstant>,
1561 ) -> Result<Vec<Frame>, RouterError> {
1562 match frame.header.ty {
1563 FrameType::Ping => Ok(vec![pong(&frame)?]),
1564 FrameType::Hello => {
1565 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1566 }
1567 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1568 FrameType::Cancel => {
1569 if self
1572 .forwarding
1573 .operator_confirms()
1574 .cancel(ctx.connection_id, frame.header.corr)
1575 || self
1576 .supervisor
1577 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1578 || self.cancel_activity_subscription(ctx, &frame)?
1579 {
1580 Ok(Vec::new())
1581 } else {
1582 Ok(vec![control_error_frame(
1583 &frame,
1584 "unknown_subscription",
1585 "no subscription has this correlation id",
1586 )?])
1587 }
1588 }
1589 FrameType::Request => {
1590 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1593 if op
1594 .as_ref()
1595 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1596 {
1597 return self.handle_activity_record(ctx, frame).await;
1598 }
1599 if op.as_ref().is_some_and(|probe| {
1600 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1601 }) {
1602 return self.handle_activity_read(ctx, frame).await;
1603 }
1604 if op
1605 .as_ref()
1606 .is_some_and(|probe| probe.op == "operator.confirm")
1607 {
1608 return self.handle_operator_confirm(ctx, frame);
1609 }
1610 if self
1611 .forwarding
1612 .module_endpoint_for_connection(ctx.connection_id)
1613 .map_err(RouterError::Forwarding)?
1614 .is_some()
1615 {
1616 if !is_known_module_request_op(&frame.body) {
1617 return Ok(vec![control_error_frame(
1618 &frame,
1619 "unsupported_control_frame",
1620 "module-originated channel-0 REQUEST is not supported",
1621 )?]);
1622 }
1623 let request = match parse_module_control_request_from_module(&frame.body) {
1624 Ok(request) => request,
1625 Err((err, ControlRequestBodyError::UnknownOp)) => {
1626 return Ok(vec![control_error_frame(
1627 &frame,
1628 "unsupported_control_frame",
1629 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1630 )?])
1631 }
1632 Err((err, ControlRequestBodyError::InvalidBody)) => {
1633 return Ok(vec![control_error_frame(
1634 &frame,
1635 "invalid_control_body",
1636 format!("malformed module control body: {err}"),
1637 )?])
1638 }
1639 };
1640 let op = module_control_request_op(&request);
1641 let corr = frame.header.corr;
1642 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1643 let result =
1644 self.handle_module_control_request(ctx.connection_id, frame, request);
1645 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1646 return result;
1647 }
1648
1649 if is_known_module_request_op(&frame.body) {
1650 return Ok(vec![control_error_frame(
1651 &frame,
1652 "not_registered",
1653 "catalog.update requires an active module registration owned by this connection",
1654 )?]);
1655 }
1656
1657 let request = match parse_client_control_request(&frame.body) {
1658 Ok(request) => request,
1659 Err((err, ControlRequestBodyError::UnknownOp)) => {
1660 return Ok(vec![control_error_frame(
1661 &frame,
1662 "unknown_control_op",
1663 format!("unknown client control op: {err}"),
1664 )?])
1665 }
1666 Err((err, ControlRequestBodyError::InvalidBody)) => {
1667 return Ok(vec![control_error_frame(
1668 &frame,
1669 "invalid_control_body",
1670 format!("malformed client control body: {err}"),
1671 )?])
1672 }
1673 };
1674 let op = client_control_request_op(&request);
1675 let corr = frame.header.corr;
1676 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1677 #[cfg(test)]
1678 if let Some(delay) = self.control_dispatch_delay {
1679 tokio::time::sleep(delay).await;
1680 }
1681 let result = self
1682 .handle_client_control_request(ctx, frame, request)
1683 .await;
1684 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1685 result
1686 }
1687 FrameType::Push => {
1688 let Some(endpoint) = self
1689 .forwarding
1690 .module_endpoint_for_connection(ctx.connection_id)
1691 .map_err(RouterError::Forwarding)?
1692 else {
1693 return Ok(vec![control_error_frame(
1694 &frame,
1695 "unsupported_control_frame",
1696 "client-originated channel-0 PUSH is not supported",
1697 )?]);
1698 };
1699 self.handle_status_update(endpoint, frame)
1700 }
1701 FrameType::Response | FrameType::Error
1702 if self
1703 .forwarding
1704 .module_endpoint_for_connection(ctx.connection_id)
1705 .map_err(RouterError::Forwarding)?
1706 .is_some() =>
1707 {
1708 self.handle_module_relay_response(ctx.connection_id, frame)
1709 }
1710 ty => Ok(vec![control_error_frame(
1711 &frame,
1712 "unsupported_control_frame",
1713 format!("unsupported channel-0 frame {ty:?}"),
1714 )?]),
1715 }
1716 }
1717
1718 pub fn cleanup_connection(
1719 &self,
1720 connection_id: ConnectionId,
1721 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1722 self.cleanup_connection_with_end_reason(
1723 connection_id,
1724 RegistrationEndReason::ConnectionClosed,
1725 )
1726 }
1727
1728 fn cleanup_connection_with_end_reason(
1729 &self,
1730 connection_id: ConnectionId,
1731 requested_reason: RegistrationEndReason,
1732 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1733 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1734 self.registry
1735 .get_module_by_connection(connection_id)?
1736 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1737 .and_then(|module| module.registration_end_reason().ok().flatten())
1738 .unwrap_or(requested_reason)
1739 } else {
1740 requested_reason
1741 };
1742 let crash_closed = self
1743 .registry
1744 .get_module_by_connection(connection_id)?
1745 .and_then(|registration| {
1746 self.forwarding
1747 .module_endpoint_for_connection(connection_id)
1748 .ok()
1749 .flatten()
1750 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1751 .map(|routes| (registration.manifest.module_id, routes))
1752 });
1753 let crash_closed = crash_closed.map(|(module_id, routes)| {
1754 let terminal = match self.supervisor.get(&module_id) {
1755 None => false,
1756 Some(module) => match module.will_recover_after_connection_loss() {
1757 Ok(will_recover) => !will_recover,
1758 Err(err) => {
1759 warn!(
1760 %module_id,
1761 error = %err,
1762 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1763 );
1764 false
1765 }
1766 },
1767 };
1768 let (reason, drained) = match self
1772 .forwarding
1773 .daemon_drain_result_for_connection(connection_id)
1774 {
1775 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
1776 Ok(None) => (RouteCloseReason::Crash, false),
1777 Err(err) => {
1778 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1779 (RouteCloseReason::Crash, false)
1780 }
1781 };
1782 (module_id, routes, reason, drained, terminal)
1783 });
1784 let registrations = self.deregister_connection(connection_id, end_reason);
1785 let cleanup = if crash_closed.is_some() {
1786 self.forwarding.cleanup_connection_counted(connection_id)
1787 } else {
1788 self.forwarding
1794 .cleanup_connection(connection_id)
1795 .map(|released| crate::forwarding::ConnectionCleanup {
1796 released,
1797 abandoned_relays: 0,
1798 })
1799 };
1800 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
1805 let abandoned = cleanup
1806 .as_ref()
1807 .map_or(0, |cleanup| cleanup.abandoned_relays);
1808 send_route_control_pushes(
1809 &self.forwarding,
1810 routes,
1811 ClientControlPush::RouteClosed {
1812 module_id,
1813 channels: Vec::new(),
1814 reason,
1815 drained,
1816 abandoned,
1817 excluded_subscriptions: 0,
1818 terminal: Some(terminal),
1819 },
1820 );
1821 }
1822 if let Ok(cleanup) = cleanup {
1823 self.emit_route_goodbyes(cleanup.released);
1824 }
1825 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1829 crate::supervise::notify_registration_release();
1830 self.capability_evaluator.wake_deadline_loop();
1831 self.refresh_capability_requirements();
1832 }
1833 self.supervisor.remove_spawn_subscribers(connection_id);
1834 self.activity_streams
1835 .lock()
1836 .unwrap_or_else(|p| p.into_inner())
1837 .retain(|(id, _), task| {
1838 if *id == connection_id {
1839 task.abort();
1840 false
1841 } else {
1842 true
1843 }
1844 });
1845 self.hello_launch_nonces
1848 .lock()
1849 .unwrap_or_else(|poisoned| poisoned.into_inner())
1850 .forget(connection_id);
1851 self.scopes
1852 .write()
1853 .unwrap_or_else(|poisoned| poisoned.into_inner())
1854 .release_connection(connection_id);
1855 registrations
1856 }
1857
1858 pub(crate) fn handle_route_goodbye(
1859 &self,
1860 connection_id: ConnectionId,
1861 route_channel: u16,
1862 route_epoch: u32,
1863 ) -> Result<bool, RouterError> {
1864 debug!(
1865 connection_id = connection_id.get(),
1866 route_channel, route_epoch, "handling route GOODBYE"
1867 );
1868 let RouteRelease::Removed(released_route) = self
1869 .forwarding
1870 .release_client_route(connection_id, route_channel, route_epoch)
1871 .map_err(RouterError::Forwarding)?
1872 else {
1873 return Ok(false);
1874 };
1875 self.emit_route_goodbyes(vec![released_route]);
1876 Ok(true)
1877 }
1878
1879 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1880 for released in released_routes {
1881 let frame = match Frame::build_with_version(
1882 released.negotiated_ver,
1883 FrameType::Goodbye,
1884 control_flags(),
1885 released.channel,
1886 released.epoch,
1887 0,
1888 Vec::new(),
1889 ) {
1890 Ok(frame) => frame,
1891 Err(err) => {
1892 warn!(
1893 route_channel = released.channel,
1894 error = %err,
1895 "failed to build route GOODBYE frame"
1896 );
1897 continue;
1898 }
1899 };
1900 if !released.close_on_delivery_failure() {
1901 crate::forwarding::send_module_route_goodbye(
1902 &self.counters,
1903 &released.sink,
1904 frame,
1905 released.module_id.as_deref(),
1906 "client route released",
1907 );
1908 continue;
1909 }
1910 if let Err(err) = released.sink.try_send(frame) {
1911 warn!(
1912 target_connection_id = released.connection_id.get(),
1913 route_channel = released.channel,
1914 error = %err,
1915 "route GOODBYE was not delivered to client; closing target connection"
1916 );
1917 if self
1918 .forwarding
1919 .escalate_client_delivery_failure(
1920 released.connection_id,
1921 released.channel,
1922 released.epoch,
1923 CloseReason::new(
1924 "route_goodbye_delivery_failed",
1925 format!(
1926 "failed to enqueue route GOODBYE for channel {}: {err}",
1927 released.channel
1928 ),
1929 ),
1930 crate::forwarding::UndeliveredFrame {
1931 module_id: released.module_id.as_deref(),
1932 sink: &released.sink,
1933 },
1934 )
1935 .unwrap_or(false)
1936 {
1937 self.counters.increment_goodbye_relay_client_failed();
1938 }
1939 }
1940 }
1941 }
1942
1943 fn send_abandoned_route_bind_goodbye(
1955 &self,
1956 module_sink: &crate::FrameSink,
1957 negotiated_ver: u8,
1958 module_channel: u16,
1959 module_epoch: u32,
1960 ) {
1961 let frame = match Frame::build_with_version(
1962 negotiated_ver,
1963 FrameType::Goodbye,
1964 control_flags(),
1965 module_channel,
1966 module_epoch,
1967 0,
1968 Vec::new(),
1969 ) {
1970 Ok(frame) => frame,
1971 Err(err) => {
1972 warn!(
1973 route_channel = module_channel,
1974 error = %err,
1975 "failed to build GOODBYE for abandoned route.bind"
1976 );
1977 return;
1978 }
1979 };
1980 crate::forwarding::send_module_route_goodbye(
1981 &self.counters,
1982 module_sink,
1983 frame,
1984 None,
1985 "abandoned route.bind",
1986 );
1987 }
1988
1989 fn handle_hello(
1990 &self,
1991 connection_id: ConnectionId,
1992 sink: Option<crate::FrameSink>,
1993 frame: Frame,
1994 ) -> Result<Vec<Frame>, RouterError> {
1995 debug!(
1996 connection_id = connection_id.get(),
1997 corr = frame.header.corr,
1998 "handling HELLO"
1999 );
2000 if self
2004 .registry
2005 .get_module_by_connection(connection_id)
2006 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2007 .is_some()
2008 {
2009 return Ok(vec![control_error_frame(
2010 &frame,
2011 "invalid_hello",
2012 "connection is already registered as a module",
2013 )?]);
2014 }
2015 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2016 Ok(value) => value,
2017 Err(err) => {
2018 return Ok(vec![control_error_frame(
2019 &frame,
2020 "invalid_hello",
2021 format!("malformed HELLO body: {err}"),
2022 )?])
2023 }
2024 };
2025 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2026 return Ok(vec![control_error_frame(
2027 &frame,
2028 "invalid_capability_grammar",
2029 err.to_string(),
2030 )?]);
2031 }
2032 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2033 return Ok(vec![control_error_frame(
2034 &frame,
2035 "invalid_manifest",
2036 err.to_string(),
2037 )?]);
2038 }
2039 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2040 return Ok(vec![control_error_frame(
2041 &frame,
2042 "invalid_event_declaration",
2043 err.to_string(),
2044 )?]);
2045 }
2046 if let Some(provenance) = hello_value
2047 .get("manifest")
2048 .and_then(|manifest| manifest.get("provenance"))
2049 {
2050 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2051 return Ok(vec![control_error_frame(
2052 &frame,
2053 "invalid_manifest",
2054 format!("malformed manifest provenance: {err}"),
2055 )?]);
2056 }
2057 }
2058 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2059 Ok(hello) => hello,
2060 Err(err) => {
2061 return Ok(vec![control_error_frame(
2062 &frame,
2063 "invalid_hello",
2064 format!("malformed HELLO body: {err}"),
2065 )?])
2066 }
2067 };
2068
2069 if hello.protocol_ver != hello.manifest.protocol_ver {
2070 return Ok(vec![control_error_frame(
2071 &frame,
2072 "invalid_manifest",
2073 format!(
2074 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2075 hello.protocol_ver, hello.manifest.protocol_ver
2076 ),
2077 )?]);
2078 }
2079
2080 if hello.manifest.module_id.trim().is_empty() {
2081 return Ok(vec![control_error_frame(
2082 &frame,
2083 "invalid_manifest",
2084 "manifest module_id must not be empty",
2085 )?]);
2086 }
2087
2088 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2089 Ok(negotiated_ver) => negotiated_ver,
2090 Err(message) => {
2091 return Ok(vec![control_error_frame(
2092 &frame,
2093 "version_unsupported",
2094 message,
2095 )?])
2096 }
2097 };
2098
2099 let swap_admission = self
2107 .supervisor
2108 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2109 if swap_admission == SwapHelloAdmission::Refused {
2110 warn!(
2111 module_id = %hello.manifest.module_id,
2112 connection_id = connection_id.get(),
2113 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2114 );
2115 return Ok(vec![control_error_frame(
2116 &frame,
2117 "swap_token_invalid",
2118 format!(
2119 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2120 hello.manifest.module_id
2121 ),
2122 )?]);
2123 }
2124 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2125
2126 if let Some(rejection) = (!swap_candidate)
2134 .then(|| {
2135 self.supervisor.reserved_hello_rejection(
2136 &hello.manifest.module_id,
2137 hello.launch_nonce.as_deref(),
2138 )
2139 })
2140 .flatten()
2141 {
2142 let message = match rejection {
2143 ReservedHelloRejection::Exact { module_id } => format!(
2144 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2145 ),
2146 ReservedHelloRejection::Prefix {
2147 prefix,
2148 owner_module_id,
2149 } => format!(
2150 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2151 hello.manifest.module_id
2152 ),
2153 };
2154 return Ok(vec![control_error_frame(
2155 &frame,
2156 "reserved_module",
2157 message,
2158 )?]);
2159 }
2160
2161 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2162 &hello.manifest.module_id,
2163 hello.manifest.capabilities.as_ref(),
2164 );
2165 if let Some(refusal) = reserved_capability_refusals.first() {
2166 let capability = refusal.capability.clone();
2167 let bound_module = refusal.claimants[0].clone();
2168 log_duplicate_claim_events(reserved_capability_refusals);
2169 return Ok(vec![control_error_frame(
2170 &frame,
2171 "reserved_capability",
2172 format!(
2173 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2174 capability, bound_module, hello.manifest.module_id
2175 ),
2176 )?]);
2177 }
2178
2179 if self
2182 .forwarding
2183 .connection_has_client_routes(connection_id)
2184 .map_err(RouterError::Forwarding)?
2185 {
2186 return Ok(vec![control_error_frame(
2187 &frame,
2188 "invalid_hello",
2189 "connection has open client routes and cannot also register as a module",
2190 )?]);
2191 }
2192
2193 self.hello_launch_nonces
2198 .lock()
2199 .unwrap_or_else(|poisoned| poisoned.into_inner())
2200 .record(connection_id, hello.launch_nonce.as_deref());
2201 let control_ops = effective_module_control_ops(hello.control_ops);
2202 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2205 if swap_candidate {
2206 return self.register_swap_candidate(
2207 connection_id,
2208 sink,
2209 &frame,
2210 hello.manifest,
2211 negotiated_ver,
2212 control_ops,
2213 hello_ack,
2214 );
2215 }
2216 let registration = match self.registry.register_with_control_ops(
2217 hello.manifest,
2218 negotiated_ver,
2219 connection_id,
2220 control_ops,
2221 ) {
2222 Ok(registration) => registration,
2223 Err(RegistryError::DuplicateModuleId { module_id }) => {
2224 return Ok(vec![control_error_frame(
2225 &frame,
2226 "duplicate_module_id",
2227 format!(
2228 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2229 ),
2230 )?])
2231 }
2232 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2233 return Ok(vec![control_error_frame(
2234 &frame,
2235 "invalid_module_id",
2236 err.to_string(),
2237 )?])
2238 }
2239 Err(err) => {
2240 return Ok(vec![control_error_frame(
2241 &frame,
2242 "registry_error",
2243 err.to_string(),
2244 )?])
2245 }
2246 };
2247
2248 let reply = if let Some(sink) = sink {
2249 let concurrency = manifest_concurrency(®istration.manifest);
2265 if let Err(err) = self.forwarding.register_module_connection_acked(
2266 connection_id,
2267 registration.manifest.module_id.clone(),
2268 negotiated_ver,
2269 concurrency,
2270 sink,
2271 hello_ack,
2272 ) {
2273 if matches!(
2277 self.deregister_connection(
2278 connection_id,
2279 RegistrationEndReason::RegistrationFailed,
2280 ),
2281 Ok(r) if !r.is_empty()
2282 ) {
2283 crate::supervise::notify_registration_release();
2284 }
2285 return Ok(vec![control_error_frame(
2286 &frame,
2287 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2288 "invalid_hello"
2289 } else {
2290 forwarding_error_code(&err)
2291 },
2292 err.to_string(),
2293 )?]);
2294 }
2295 Vec::new()
2296 } else {
2297 vec![hello_ack]
2300 };
2301
2302 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2311 info!(
2312 module_id = %registration.manifest.module_id,
2313 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2314 );
2315 }
2316
2317 self.apply_registration_capabilities(®istration);
2318
2319 self.supervisor
2320 .record_activity_registration(®istration.manifest.module_id);
2321
2322 info!(
2323 module_id = %registration.manifest.module_id,
2324 module_version = %registration.manifest.module_version,
2325 negotiated_ver,
2326 routable_provider = manifest_provides_routable_role(®istration.manifest),
2327 connection_id = connection_id.get(),
2328 "module registered"
2329 );
2330
2331 Ok(reply)
2332 }
2333
2334 #[allow(clippy::too_many_arguments)]
2344 fn register_swap_candidate(
2345 &self,
2346 connection_id: ConnectionId,
2347 sink: Option<crate::FrameSink>,
2348 frame: &Frame,
2349 manifest: ModuleManifest,
2350 negotiated_ver: u8,
2351 control_ops: Vec<String>,
2352 hello_ack: Frame,
2353 ) -> Result<Vec<Frame>, RouterError> {
2354 let module_id = manifest.module_id.clone();
2355 let registration = match self.registry.register_candidate_with_control_ops(
2356 manifest,
2357 negotiated_ver,
2358 connection_id,
2359 control_ops,
2360 ) {
2361 Ok(registration) => registration,
2362 Err(RegistryError::DuplicateModuleId { module_id }) => {
2363 return Ok(vec![control_error_frame(
2364 frame,
2365 "duplicate_module_id",
2366 format!(
2367 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2368 ),
2369 )?])
2370 }
2371 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2372 return Ok(vec![control_error_frame(
2373 frame,
2374 "invalid_module_id",
2375 err.to_string(),
2376 )?])
2377 }
2378 Err(err) => {
2379 return Ok(vec![control_error_frame(
2380 frame,
2381 "registry_error",
2382 err.to_string(),
2383 )?])
2384 }
2385 };
2386 let reply = if let Some(sink) = sink {
2387 let concurrency = manifest_concurrency(®istration.manifest);
2391 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2392 connection_id,
2393 module_id.clone(),
2394 negotiated_ver,
2395 concurrency,
2396 sink,
2397 hello_ack,
2398 ) {
2399 if matches!(
2400 self.deregister_connection(
2401 connection_id,
2402 RegistrationEndReason::RegistrationFailed,
2403 ),
2404 Ok(r) if !r.is_empty()
2405 ) {
2406 crate::supervise::notify_registration_release();
2407 }
2408 return Ok(vec![control_error_frame(
2409 frame,
2410 forwarding_error_code(&err),
2411 err.to_string(),
2412 )?]);
2413 }
2414 Vec::new()
2415 } else {
2416 vec![hello_ack]
2417 };
2418 self.supervisor.mark_swap_candidate_admitted(&module_id);
2419 info!(
2420 module_id = %module_id,
2421 module_version = %registration.manifest.module_version,
2422 negotiated_ver,
2423 ready = registration.ready,
2424 connection_id = connection_id.get(),
2425 "swap candidate registered; not routable until cutover"
2426 );
2427 Ok(reply)
2428 }
2429
2430 fn build_hello_ack(
2431 &self,
2432 frame: &Frame,
2433 negotiated_ver: u8,
2434 module_id: &str,
2435 ) -> Result<Frame, RouterError> {
2436 let ack = ModuleHelloAckBody {
2437 negotiated_ver,
2438 subc_ops: module_subc_ops(),
2439 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2440 storage: self
2441 .storage_config
2442 .as_ref()
2443 .map(|cfg| cfg.descriptor_for(module_id)),
2444 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2445 };
2446 let body = serde_json::to_vec(&ack).map_err(|err| {
2447 RouterError::backend(
2448 0,
2449 frame.header.corr,
2450 format!("failed to encode HELLO_ACK: {err}"),
2451 )
2452 })?;
2453
2454 Frame::build_with_version(
2455 negotiated_ver,
2456 FrameType::HelloAck,
2457 control_flags(),
2458 0,
2459 0,
2460 frame.header.corr,
2461 body,
2462 )
2463 .map_err(RouterError::FrameBuild)
2464 }
2465
2466 async fn handle_client_control_request(
2467 &self,
2468 ctx: &RouteCtx,
2469 frame: Frame,
2470 request: ClientControlRequest,
2471 ) -> Result<Vec<Frame>, RouterError> {
2472 match request {
2473 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2474 ClientControlRequest::CatalogList { module_id } => {
2475 self.handle_catalog_list(frame, module_id)
2476 }
2477 ClientControlRequest::RouteOpen {
2478 target,
2479 identity,
2480 consumer_identity,
2481 consumer_capabilities,
2482 role_versions,
2483 admission_facts,
2484 scope,
2485 } => {
2486 self.handle_route_open(
2487 ctx,
2488 frame,
2489 RouteOpenRequest {
2490 target,
2491 identity,
2492 consumer_identity,
2493 consumer_capabilities,
2494 role_versions,
2495 admission_facts,
2496 scope,
2497 },
2498 )
2499 .await
2500 }
2501 ClientControlRequest::RoutePoll {
2502 route_channel,
2503 route_epoch,
2504 kind,
2505 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2506 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2507 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2508 self.handle_supervisor_spawn_snapshot(frame)
2509 }
2510 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2511 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2512 }
2513 ClientControlRequest::SupervisorRestart {
2514 module_id,
2515 drain_timeout_ms,
2516 } => {
2517 self.log_supervisor_request_received(
2518 ctx,
2519 frame.header.corr,
2520 ops::SUPERVISOR_RESTART,
2521 Some(&module_id),
2522 None,
2523 )?;
2524 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2525 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2526 .await
2527 }
2528 ClientControlRequest::SupervisorSwap {
2529 module_id,
2530 ready_timeout_ms,
2531 } => {
2532 self.log_supervisor_request_received(
2533 ctx,
2534 frame.header.corr,
2535 ops::SUPERVISOR_SWAP,
2536 Some(&module_id),
2537 None,
2538 )?;
2539 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2540 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2541 .await
2542 }
2543 ClientControlRequest::SupervisorReload { module_id } => {
2544 self.log_supervisor_request_received(
2545 ctx,
2546 frame.header.corr,
2547 ops::SUPERVISOR_RELOAD,
2548 Some(&module_id),
2549 None,
2550 )?;
2551 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2552 self.handle_supervisor_reload(frame, module_id, requester)
2553 .await
2554 }
2555 ClientControlRequest::SupervisorRescan { preview } => {
2556 if !preview {
2557 self.log_supervisor_request_received(
2558 ctx,
2559 frame.header.corr,
2560 ops::SUPERVISOR_RESCAN,
2561 None,
2562 None,
2563 )?;
2564 }
2565 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2566 self.handle_supervisor_rescan(frame, preview, requester)
2567 .await
2568 }
2569 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2570 self.log_supervisor_request_received(
2571 ctx,
2572 frame.header.corr,
2573 ops::SUPERVISOR_RELEASE_RESERVED,
2574 Some(&module_id),
2575 None,
2576 )?;
2577 self.handle_supervisor_release_reserved(frame, module_id)
2578 .await
2579 }
2580 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2581 self.log_supervisor_request_received(
2582 ctx,
2583 frame.header.corr,
2584 ops::SUPERVISOR_SET_ENABLED,
2585 Some(&module_id),
2586 Some(enabled),
2587 )?;
2588 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2589 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2590 .await
2591 }
2592 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2593 self.handle_supervisor_health_probe(frame, module_id).await
2594 }
2595 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2596 ClientControlRequest::SupervisorRoutes { module_id } => {
2597 self.handle_supervisor_routes(frame, module_id)
2598 }
2599 ClientControlRequest::SupervisorProvenance { module_id } => {
2600 self.handle_supervisor_provenance(frame, module_id).await
2601 }
2602 ClientControlRequest::SupervisorStderrTail {
2603 module_id,
2604 max_lines,
2605 max_bytes,
2606 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2607 ClientControlRequest::SupervisorTerminals { module_id } => {
2608 self.handle_supervisor_terminals(frame, module_id).await
2609 }
2610 }
2611 }
2612
2613 fn log_supervisor_request_received(
2614 &self,
2615 ctx: &RouteCtx,
2616 corr: u64,
2617 op: &'static str,
2618 module_id: Option<&str>,
2619 enabled: Option<bool>,
2620 ) -> Result<(), RouterError> {
2621 let caller = self.supervisor_requester(ctx, corr)?;
2622 let caller = principal_label(&caller);
2623 info!(
2626 target: "control",
2627 op,
2628 module_id,
2629 enabled,
2630 connection_id = ctx.connection_id.get(),
2631 caller = %caller,
2632 "supervisor request received"
2633 );
2634 Ok(())
2635 }
2636
2637 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2638 Ok(self
2639 .registry
2640 .get_module_by_connection(ctx.connection_id)
2641 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2642 .map(|registration| Principal::Reserved {
2643 module_id: registration.manifest.module_id,
2644 })
2645 .unwrap_or(Principal::Direct))
2646 }
2647
2648 fn handle_module_control_request(
2649 &self,
2650 connection_id: ConnectionId,
2651 frame: Frame,
2652 request: ModuleControlRequestFromModule,
2653 ) -> Result<Vec<Frame>, RouterError> {
2654 match request {
2655 ModuleControlRequestFromModule::CatalogUpdate {
2656 provides,
2657 capabilities,
2658 ready,
2659 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2660 ModuleControlRequestFromModule::LiveRoots {} => {
2661 let registered = self
2662 .registry
2663 .get_module_by_connection(connection_id)
2664 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2665 let Some(registration) = registered else {
2666 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2667 };
2668 let response = self
2669 .forwarding
2670 .live_roots(®istration.manifest.module_id)
2671 .map_err(RouterError::Forwarding)?;
2672 Ok(vec![control_response_body_frame(
2673 &frame,
2674 &response,
2675 "ModuleControlResponseToModule::LiveRoots",
2676 )?])
2677 }
2678 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2679 self.handle_scope_sync(connection_id, frame, generation, scopes)
2680 }
2681 ModuleControlRequestFromModule::ScopeApply {
2682 generation,
2683 upsert,
2684 end,
2685 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2686 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2687 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2688 }
2689 }
2690 }
2691
2692 fn handle_operator_confirm(
2693 &self,
2694 ctx: &RouteCtx,
2695 frame: Frame,
2696 ) -> Result<Vec<Frame>, RouterError> {
2697 use crate::operator_confirm::{audit, Outcome};
2698 let registration = self
2699 .registry
2700 .get_module_by_connection(ctx.connection_id)
2701 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2702 let Some(registration) = registration else {
2703 let outcome = Outcome::refusal("not_registered");
2704 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2705 return Ok(vec![outcome.frame(&frame)]);
2706 };
2707 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2708 Ok(request) => request,
2709 Err(_) => {
2710 let outcome = Outcome::refusal("invalid_control_body");
2711 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2712 return Ok(vec![outcome.frame(&frame)]);
2713 }
2714 };
2715 let module_id = registration.manifest.module_id;
2716 let nonce = self
2720 .hello_launch_nonces
2721 .lock()
2722 .unwrap_or_else(|p| p.into_inner())
2723 .nonce(ctx.connection_id)
2724 .map(str::to_owned);
2725 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2726 self.supervisor
2727 .spawned_consumer_authorized(&module_id, nonce)
2728 });
2729 let confirms = self.forwarding.operator_confirms();
2730 self.forwarding
2731 .with_operator_route(
2732 ctx.connection_id,
2733 request.route_channel,
2734 request.route_epoch,
2735 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
2736 )
2737 .map_err(RouterError::Forwarding)
2738 }
2739
2740 fn handle_scope_sync(
2744 &self,
2745 connection_id: ConnectionId,
2746 frame: Frame,
2747 generation: u64,
2748 scopes: Vec<ScopeRecord>,
2749 ) -> Result<Vec<Frame>, RouterError> {
2750 self.handle_scope_change(connection_id, frame, generation, scopes, None)
2751 }
2752
2753 fn handle_scope_change(
2754 &self,
2755 connection_id: ConnectionId,
2756 frame: Frame,
2757 generation: u64,
2758 scopes: Vec<ScopeRecord>,
2759 end: Option<Vec<ScopeEnd>>,
2760 ) -> Result<Vec<Frame>, RouterError> {
2761 let is_apply = end.is_some();
2762 let Some(registration) = self
2763 .registry
2764 .get_module_by_connection(connection_id)
2765 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2766 else {
2767 return Ok(vec![control_error_frame(
2768 &frame,
2769 "not_registered",
2770 "scope.sync and scope.apply require an active module registration owned by this connection",
2771 )?]);
2772 };
2773 let owner = registration.manifest.module_id;
2774 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2775 let is_current_launch = |connection: ConnectionId| {
2776 self.hello_launch_nonces
2777 .lock()
2778 .unwrap_or_else(|poisoned| poisoned.into_inner())
2779 .presented(connection, current_nonce.as_deref())
2780 };
2781 let mut table = self
2786 .scopes
2787 .write()
2788 .unwrap_or_else(|poisoned| poisoned.into_inner());
2789 let outcome = match end {
2790 Some(end) => table.apply(
2791 &owner,
2792 connection_id,
2793 is_current_launch,
2794 generation,
2795 scopes,
2796 end,
2797 ),
2798 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
2799 };
2800 let drained = match &outcome {
2801 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
2802 Err(_) => Vec::new(),
2803 };
2804 let has_deadlines = outcome.is_ok() && table.has_deadlines();
2805 drop(table);
2806 if has_deadlines {
2807 self.scope_deadline_added.notify_one();
2808 }
2809 match outcome {
2810 Ok(applied) => {
2811 let counts = ScopeOutcomeCounts::of(&applied.results);
2812 info!(
2813 owner = %owner,
2814 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2815 generation,
2816 records = applied.results.len(),
2817 created = counts.created,
2818 replaced = counts.replaced,
2819 updated = counts.updated,
2820 unchanged = counts.unchanged,
2821 refused = counts.refused,
2822 ended = applied.ended.len(),
2823 tag_changes = applied.tag_changes.len(),
2824 routes_closed = drained.len(),
2825 "scope change accepted"
2826 );
2827 for refused in applied
2833 .results
2834 .iter()
2835 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2836 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2837 {
2838 warn!(
2839 owner = %owner,
2840 generation,
2841 scope_ref = %refused.scope_ref,
2842 scope_epoch = refused.scope_epoch,
2843 code = refused.code.as_deref().unwrap_or(""),
2844 "scope record refused"
2845 );
2846 }
2847 self.close_scope_drained_routes(drained);
2848 let response = if is_apply {
2849 ModuleControlResponseToModule::ScopeApply {
2850 generation,
2851 results: applied.results,
2852 end_results: applied.end_results,
2853 ended: applied.ended,
2854 }
2855 } else {
2856 ModuleControlResponseToModule::ScopeSync {
2857 generation,
2858 results: applied.results,
2859 ended: applied.ended,
2860 }
2861 };
2862 Ok(vec![control_response_body_frame(
2863 &frame,
2864 &response,
2865 "scope change response",
2866 )?])
2867 }
2868 Err(refusal) => {
2869 info!(
2870 owner = %owner,
2871 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2872 generation,
2873 code = refusal.code,
2874 "scope change refused"
2875 );
2876 Ok(vec![control_error_frame(
2877 &frame,
2878 refusal.code,
2879 refusal.message,
2880 )?])
2881 }
2882 }
2883 }
2884
2885 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2891 if drained.is_empty() {
2892 return;
2893 }
2894 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2895 BTreeMap::new();
2896 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2897 for route in drained {
2898 warn!(
2899 module_id = %route.module_id,
2900 reason = ?route.reason,
2901 client_connection_id = route.client.connection_id.get(),
2902 route_channel = route.client.channel,
2903 "closing route because its scope changed"
2904 );
2905 pushes
2906 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2907 .or_insert_with(|| (route.reason, Vec::new()))
2908 .1
2909 .push(EndpointRoute {
2910 goodbye_target: route.client.clone(),
2911 principal: Principal::Unverified,
2912 bound_at: Instant::now(),
2913 draining: false,
2914 drain_reason: None,
2915 });
2916 goodbyes.push(route.module);
2917 goodbyes.push(route.client);
2918 }
2919 for ((module_id, _), (reason, routes)) in pushes {
2920 send_route_control_pushes(
2921 &self.forwarding,
2922 routes,
2923 ClientControlPush::RouteClosed {
2924 module_id,
2925 channels: Vec::new(),
2926 reason,
2927 drained: false,
2928 abandoned: 0,
2929 excluded_subscriptions: 0,
2930 terminal: Some(false),
2931 },
2932 );
2933 }
2934 self.emit_route_goodbyes(goodbyes);
2935 }
2936
2937 fn handle_scope_describe(
2940 &self,
2941 connection_id: ConnectionId,
2942 frame: Frame,
2943 owner: Principal,
2944 scope_ref: String,
2945 ) -> Result<Vec<Frame>, RouterError> {
2946 let registered = self
2947 .registry
2948 .get_module_by_connection(connection_id)
2949 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2950 if registered.is_none() {
2951 return Ok(vec![control_error_frame(
2952 &frame,
2953 "not_registered",
2954 "scope.describe requires an active module registration owned by this connection",
2955 )?]);
2956 }
2957 let description = self
2958 .scopes
2959 .read()
2960 .unwrap_or_else(|poisoned| poisoned.into_inner())
2961 .describe(&owner, &scope_ref);
2962 let owner_configured = match &owner {
2963 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
2968 _ => false,
2969 };
2970 let response = ModuleControlResponseToModule::ScopeDescribe {
2971 status: description.status,
2972 scope_epoch: description.scope_epoch,
2973 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2974 owner_synced: description.owner_synced,
2975 owner_configured,
2976 scope: description.stamp,
2977 };
2978 Ok(vec![control_response_body_frame(
2979 &frame,
2980 &response,
2981 "ModuleControlResponseToModule::ScopeDescribe",
2982 )?])
2983 }
2984
2985 fn handle_catalog_update(
2986 &self,
2987 connection_id: ConnectionId,
2988 frame: Frame,
2989 provides: Vec<ProviderRole>,
2990 capabilities: Option<CapabilityDeclarations>,
2991 ready: Option<bool>,
2992 ) -> Result<Vec<Frame>, RouterError> {
2993 self.refresh_capability_requirements();
2994 let Some(registration) = self
2995 .registry
2996 .get_module_by_connection(connection_id)
2997 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2998 else {
2999 return Ok(vec![control_error_frame(
3000 &frame,
3001 "not_registered",
3002 "catalog.update requires an active module registration owned by this connection",
3003 )?]);
3004 };
3005
3006 if let Some(message) =
3007 catalog_update_frozen_field_message(®istration.manifest, &provides)
3008 {
3009 return Ok(vec![control_error_frame(
3010 &frame,
3011 "catalog_update_frozen_field",
3012 message,
3013 )?]);
3014 }
3015
3016 let mut candidate = registration.manifest.clone();
3017 candidate.provides = provides.clone();
3018 candidate.capabilities = capabilities
3019 .clone()
3020 .or_else(|| registration.manifest.capabilities.clone());
3021 if let Err(err) = candidate.validate_capability_grammar() {
3022 return Ok(vec![control_error_frame(
3023 &frame,
3024 "invalid_capability_grammar",
3025 err.to_string(),
3026 )?]);
3027 }
3028
3029 let mut conflicts = self
3032 .capability_evaluator
3033 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3034 if let Some(conflict) = conflicts.first() {
3035 let message = format!(
3036 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3037 conflict.capability, conflict.claimants[0], candidate.module_id
3038 );
3039 for conflict in &mut conflicts {
3040 conflict.source = DuplicateClaimSource::CatalogUpdate;
3041 }
3042 log_duplicate_claim_events(conflicts);
3043 return Ok(vec![control_error_frame(
3044 &frame,
3045 "reserved_capability",
3046 message,
3047 )?]);
3048 }
3049
3050 let updated = self
3051 .registry
3052 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3053 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3054 if updated.is_none() {
3055 return Ok(vec![control_error_frame(
3056 &frame,
3057 "not_registered",
3058 "catalog.update requires an active module registration owned by this connection",
3059 )?]);
3060 }
3061 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3062 log_duplicate_claim_events(
3063 self.capability_evaluator
3064 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3065 );
3066 }
3067 if capability_census_trigger(
3068 registration.manifest.capabilities.as_ref(),
3069 updated
3070 .as_ref()
3071 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3072 ) {
3073 self.enforce_capability_denies();
3074 }
3075 self.refresh_capability_requirements();
3076
3077 let response = ModuleControlResponseToModule::CatalogUpdate {};
3078 control_response_body_frame(
3079 &frame,
3080 &response,
3081 "ModuleControlResponseToModule::CatalogUpdate",
3082 )
3083 .map(|frame| vec![frame])
3084 }
3085
3086 pub(crate) fn with_activity_ledger_config(
3087 mut self,
3088 run_dir: &Path,
3089 incarnation: String,
3090 config: crate::activity::ActivityConfig,
3091 ) -> Self {
3092 self.activity = Some(crate::activity::ActivityLedger::open_config(
3093 run_dir.join("activity.jsonl"),
3094 incarnation,
3095 config,
3096 ));
3097 self.supervisor.set_activity(self.activity.clone());
3098 self
3099 }
3100
3101 pub(crate) async fn configure_activity(
3102 &self,
3103 config: crate::activity::ActivityConfig,
3104 ) -> Result<(), &'static str> {
3105 match &self.activity {
3106 Some(ledger) => ledger.configure(config).await,
3107 None => Ok(()),
3108 }
3109 }
3110
3111 fn cancel_activity_subscription(
3112 &self,
3113 ctx: &RouteCtx,
3114 frame: &Frame,
3115 ) -> Result<bool, RouterError> {
3116 let task = self
3117 .activity_streams
3118 .lock()
3119 .unwrap_or_else(|p| p.into_inner())
3120 .remove(&(ctx.connection_id, frame.header.corr));
3121 let Some(task) = task else {
3122 return Ok(false);
3123 };
3124 task.abort();
3125 let end = Frame::build_with_version(
3126 response_version(frame),
3127 FrameType::StreamEnd,
3128 control_flags(),
3129 0,
3130 0,
3131 frame.header.corr,
3132 Vec::new(),
3133 )
3134 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3135 let sink = ctx.egress.clone();
3136 tokio::spawn(async move {
3137 let _ = sink.send(end).await;
3138 });
3139 Ok(true)
3140 }
3141
3142 async fn handle_activity_read(
3143 &self,
3144 ctx: &RouteCtx,
3145 frame: Frame,
3146 ) -> Result<Vec<Frame>, RouterError> {
3147 use subc_control::{
3148 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3149 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3150 };
3151 #[derive(Deserialize)]
3152 struct Identity {
3153 #[serde(default)]
3154 consumer_identity: Option<ConsumerIdentity>,
3155 }
3156 let principal = match self
3160 .forwarding
3161 .module_id_for_connection(ctx.connection_id)
3162 .map_err(RouterError::Forwarding)?
3163 {
3164 Some(module_id) => Ok(Principal::Reserved { module_id }),
3165 None => {
3166 let identity: Identity = match serde_json::from_slice(&frame.body) {
3167 Ok(identity) => identity,
3168 Err(_) => {
3169 return Ok(vec![control_error_frame(
3170 &frame,
3171 ACTIVITY_MALFORMED,
3172 "malformed activity read",
3173 )?])
3174 }
3175 };
3176 self.route_open_principal(&frame, identity.consumer_identity)?
3177 }
3178 };
3179 let admitted = matches!(principal, Ok(Principal::Direct))
3180 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3181 if !admitted {
3182 return Ok(vec![control_error_frame(
3183 &frame,
3184 ACTIVITY_READ_FORBIDDEN,
3185 "activity reads require an operator or trusted mirror principal",
3186 )?]);
3187 }
3188 let Some(ledger) = &self.activity else {
3189 return Ok(vec![control_error_frame(
3190 &frame,
3191 ACTIVITY_UNAVAILABLE,
3192 "activity ledger unavailable",
3193 )?]);
3194 };
3195 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3196 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3197 {
3198 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3199 Ok(request) => ledger.query(request).await,
3200 Err(_) => Err(ACTIVITY_MALFORMED),
3201 };
3202 return match result {
3203 Ok(reply) => Ok(vec![control_response_body_frame(
3204 &frame,
3205 &reply,
3206 ops::ACTIVITY_QUERY,
3207 )?]),
3208 Err(code) => Ok(vec![control_error_frame(
3209 &frame,
3210 code,
3211 "activity query refused",
3212 )?]),
3213 };
3214 }
3215 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3216 Ok(request) => request,
3217 Err(_) => {
3218 return Ok(vec![control_error_frame(
3219 &frame,
3220 ACTIVITY_MALFORMED,
3221 "malformed activity subscription",
3222 )?])
3223 }
3224 };
3225 let mut subscription = match ledger.subscribe(request.since).await {
3226 Ok(subscription) => subscription,
3227 Err(code) => {
3228 return Ok(vec![control_error_frame(
3229 &frame,
3230 code,
3231 "activity subscription refused",
3232 )?])
3233 }
3234 };
3235 let key = (ctx.connection_id, frame.header.corr);
3236 let sink = ctx.egress.clone();
3237 let mut streams = self
3238 .activity_streams
3239 .lock()
3240 .unwrap_or_else(|p| p.into_inner());
3241 streams.retain(|_, task| !task.is_finished());
3242 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3243 return Ok(vec![control_error_frame(
3244 &frame,
3245 "duplicate_subscription",
3246 "activity subscription correlation id already in use",
3247 )?]);
3248 }
3249 let task = tokio::spawn(async move {
3250 loop {
3251 let entry = if let Some(entry) = subscription.replay.pop_front() {
3252 Some(entry)
3253 } else {
3254 subscription.receiver.recv().await
3255 };
3256 let Some(entry) = entry else {
3257 break;
3258 };
3259 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3260 return;
3261 };
3262 let Ok(next) = Frame::build_with_version(
3263 response_version(&frame),
3264 FrameType::StreamData,
3265 control_flags(),
3266 0,
3267 0,
3268 frame.header.corr,
3269 body,
3270 ) else {
3271 return;
3272 };
3273 if sink.send(next).await.is_err() {
3274 return;
3275 }
3276 }
3277 if let Some(code) = subscription.terminal_code() {
3280 if let Ok(end) =
3281 control_error_frame(&frame, code, "activity subscriber fell behind")
3282 {
3283 let _ = sink.send(end).await;
3284 }
3285 }
3286 });
3287 streams.insert(key, task.abort_handle());
3288 Ok(Vec::new())
3289 }
3290
3291 async fn handle_activity_record(
3292 &self,
3293 ctx: &RouteCtx,
3294 frame: Frame,
3295 ) -> Result<Vec<Frame>, RouterError> {
3296 use subc_control::{
3297 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3298 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3299 };
3300 let result = async {
3301 let module = self
3302 .forwarding
3303 .module_id_for_connection(ctx.connection_id)
3304 .ok()
3305 .flatten()
3306 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3307 let request: ActivityRecordRequest =
3308 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3309 crate::activity::validate(&module, &request)?;
3310 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3311 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3312 let at_ms = std::time::SystemTime::now()
3313 .duration_since(std::time::UNIX_EPOCH)
3314 .unwrap_or_default()
3315 .as_millis() as u64;
3316 ledger.record(module, request, actor, at_ms).await
3317 }
3318 .await;
3319 match result {
3320 Ok(entry) => Ok(vec![control_response_body_frame(
3321 &frame,
3322 &ActivityRecordResponse { seq: entry.seq },
3323 "activity.record",
3324 )?]),
3325 Err(code) => Ok(vec![control_error_frame(
3326 &frame,
3327 code,
3328 "activity recording refused",
3329 )?]),
3330 }
3331 }
3332
3333 fn activity_actor(
3334 &self,
3335 connection: ConnectionId,
3336 module: &str,
3337 on: Option<&subc_control::ActivityOn>,
3338 ) -> Result<subc_control::ActivityActor, &'static str> {
3339 use crate::forwarding::{DataRoute, DataRouteState};
3340 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3341 let principal = |id: &str| Principal::Reserved {
3342 module_id: id.to_owned(),
3343 };
3344 let mut actor = ActivityActor {
3345 principal: principal(module),
3346 scope_owner: None,
3347 agent_id: None,
3348 run_id: None,
3349 flow_id: None,
3350 project_root: None,
3351 };
3352 let scope = match on {
3353 None => return Ok(actor),
3354 Some(ActivityOn::Scope(scope)) => {
3355 let selector = subc_protocol::scope::ScopeSelector {
3356 owner: principal(module),
3357 scope_ref: scope.scope_ref.clone(),
3358 scope_epoch: Some(scope.epoch),
3359 };
3360 self.scopes
3361 .read()
3362 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3363 .admit(&principal(module), module, &selector, true)
3364 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3365 }
3366 Some(ActivityOn::Route(route)) => {
3367 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3368 .forwarding
3369 .lookup_data_route(connection, route.channel, route.epoch)
3370 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3371 else {
3372 return Err(ACTIVITY_ACTOR_UNKNOWN);
3373 };
3374 if binding.module_id != module {
3375 return Err(ACTIVITY_ACTOR_UNKNOWN);
3376 }
3377 actor.principal = binding.principal.clone();
3378 actor.project_root = binding
3379 .project_root
3380 .as_ref()
3381 .map(|root| root.as_path().to_path_buf());
3382 let Some(bound) = &binding.scope else {
3383 return Ok(actor);
3384 };
3385 let owner = principal(&bound.owner);
3386 let selector = subc_protocol::scope::ScopeSelector {
3387 owner: owner.clone(),
3388 scope_ref: bound.scope_ref.clone(),
3389 scope_epoch: Some(bound.tag.scope_epoch),
3390 };
3391 let admitted = self
3392 .scopes
3393 .read()
3394 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3395 .admit(&owner, module, &selector, true)
3396 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3397 if admitted.tag != bound.tag {
3398 return Err(ACTIVITY_ACTOR_UNKNOWN);
3399 }
3400 admitted
3401 }
3402 };
3403 actor.scope_owner = Some(scope.stamp.owner);
3404 actor.agent_id = scope.stamp.attributes.agent_id;
3405 actor.run_id = scope.stamp.attributes.run_id;
3406 actor.flow_id = scope.stamp.attributes.flow_id;
3407 Ok(actor)
3408 }
3409
3410 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3411 self.refresh_capability_requirements();
3412 let mut counters = self.counters.snapshot();
3418 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3419 obj.insert(
3420 "activity_unparseable_lines".into(),
3421 ledger.read_errors().into(),
3422 );
3423 obj.insert(
3424 "activity_lifecycle_append_failures".into(),
3425 ledger.lifecycle_failures().into(),
3426 );
3427 obj.insert(
3428 "activity_started_digest_skips".into(),
3429 ledger.lifecycle_digest_skips().into(),
3430 );
3431 }
3432 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3433 self.forwarding.client_route_concentration(),
3434 counters.as_object_mut(),
3435 ) {
3436 obj.insert(
3437 "client_connections_with_routes".into(),
3438 connections_with_routes.into(),
3439 );
3440 obj.insert("max_routes_on_one_connection".into(), max.into());
3441 }
3442 if let (Some(open_breakers), Some(obj)) = (
3448 self.route_bind_breakers.open_snapshot(),
3449 counters.as_object_mut(),
3450 ) {
3451 obj.insert("route_bind_breakers_open".into(), open_breakers);
3452 }
3453 let response = ClientControlResponse::ServerDescribe {
3454 protocol_ver: PROTOCOL_VERSION,
3455 subc_ops: subc_ops(),
3456 capabilities: self.subc_capabilities.as_ref().to_vec(),
3457 connected_clients: self.connected_clients.count(),
3458 counters: Some(counters),
3459 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3460 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3461 capability_requirements: self.capability_requirement_statuses(),
3462 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3463 };
3464 Ok(vec![control_response_body_frame(
3465 &frame,
3466 &response,
3467 "ClientControlResponse::ServerDescribe",
3468 )?])
3469 }
3470
3471 fn handle_catalog_list(
3472 &self,
3473 frame: Frame,
3474 module_id: Option<String>,
3475 ) -> Result<Vec<Frame>, RouterError> {
3476 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3477 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3478 })?;
3479 let entries = modules
3480 .into_iter()
3481 .filter(|registration| {
3482 module_id
3483 .as_deref()
3484 .map(|wanted| registration.manifest.module_id == wanted)
3485 .unwrap_or(true)
3486 })
3487 .map(|registration| {
3488 let not_ready = self.not_ready_reason(®istration);
3489 let roles = registration.manifest.provides;
3490 CatalogEntry::new(
3491 registration.manifest.module_id,
3492 roles,
3493 registration.control_ops,
3494 )
3495 .with_ready(not_ready.is_none())
3496 .with_not_ready(not_ready)
3497 .with_module_version(Some(registration.manifest.module_version))
3498 .with_capabilities(registration.manifest.capabilities)
3499 .with_self_signals(registration.manifest.self_signals)
3500 .with_events(registration.manifest.events)
3501 })
3502 .collect();
3503 let response = ClientControlResponse::CatalogList {
3504 generation,
3505 modules: entries,
3506 subc_ops: subc_ops(),
3507 };
3508 Ok(vec![control_response_body_frame(
3509 &frame,
3510 &response,
3511 "ClientControlResponse::CatalogList",
3512 )?])
3513 }
3514
3515 fn route_open_principal(
3516 &self,
3517 frame: &Frame,
3518 consumer_identity: Option<ConsumerIdentity>,
3519 ) -> Result<Result<Principal, Frame>, RouterError> {
3520 let Some(consumer_identity) = consumer_identity else {
3521 return Ok(Ok(Principal::Direct));
3522 };
3523
3524 if self.supervisor.spawned_consumer_authorized(
3525 &consumer_identity.module_id,
3526 &consumer_identity.launch_nonce,
3527 ) {
3528 return Ok(Ok(Principal::Reserved {
3529 module_id: consumer_identity.module_id,
3530 }));
3531 }
3532
3533 Ok(Err(control_error_frame(
3534 frame,
3535 "bad_consumer_identity",
3536 format!(
3537 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3538 consumer_identity.module_id
3539 ),
3540 )?))
3541 }
3542
3543 fn route_open_refusal_frame(
3550 &self,
3551 ctx: &RouteCtx,
3552 frame: &Frame,
3553 module_id: &str,
3554 reason: &'static str,
3555 code: &'static str,
3556 message: impl Into<String>,
3557 ) -> Result<Frame, RouterError> {
3558 self.observe_route_open_refusal(ctx, module_id, reason, code);
3559 control_error_frame(frame, code, message.into())
3560 }
3561
3562 fn route_open_breaker_refusal_frame(
3580 &self,
3581 ctx: &RouteCtx,
3582 frame: &Frame,
3583 module_id: &str,
3584 consecutive_timeouts: u32,
3585 retry_in: Duration,
3586 probe_in_flight: bool,
3587 ) -> Result<Frame, RouterError> {
3588 self.counters
3589 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3590 debug!(
3591 target: "control",
3592 code = "module_timeout",
3593 module_id = ?module_id,
3594 connection_id = ctx.connection_id.get(),
3595 consecutive_timeouts,
3596 retry_in_ms = retry_in.as_millis() as u64,
3597 probe_in_flight,
3598 "route.open refused by open bind-relay breaker"
3599 );
3600 let detail = if probe_in_flight {
3607 "one test route is already being tried; retry once it settles".to_string()
3608 } else {
3609 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3610 };
3611 control_error_frame(
3612 frame,
3613 "module_timeout",
3614 format!(
3615 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3616 timed out in a row); {detail}; its established routes are unaffected"
3617 ),
3618 )
3619 }
3620
3621 fn observe_route_open_refusal(
3634 &self,
3635 ctx: &RouteCtx,
3636 module_id: &str,
3637 reason: &'static str,
3638 code: &'static str,
3639 ) {
3640 self.counters.increment_route_open_refused(code);
3641 info!(
3642 target: "control",
3643 code,
3644 reason,
3645 module_id = ?module_id,
3646 connection_id = ctx.connection_id.get(),
3647 "route.open refused"
3648 );
3649 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3650 self.route_outages.record_not_serving(module_id, reason);
3651 }
3652 }
3653
3654 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3699 self.route_outages.record_accepted(module_id);
3700 self.counters.increment_route_open_accepted(principal);
3701 info!(
3702 target: "control",
3703 principal,
3704 module_id,
3705 connection_id = ctx.connection_id.get(),
3706 "route.open accepted"
3707 );
3708 }
3709
3710 fn supervised_absent_route_open_refusal_frame(
3711 &self,
3712 ctx: &RouteCtx,
3713 frame: &Frame,
3714 module_id: &str,
3715 code: &'static str,
3716 status: &crate::supervise::ModuleStatus,
3717 ) -> Result<Frame, RouterError> {
3718 self.counters.increment_route_open_refused(code);
3719 info!(
3720 target: "control",
3721 code,
3722 reason = "supervised_not_registered",
3723 module_id = ?module_id,
3724 connection_id = ctx.connection_id.get(),
3725 state = %status.state,
3726 enabled = status.enabled,
3727 live = status.live,
3728 "route.open refused"
3729 );
3730 self.route_outages
3734 .record_not_serving(module_id, "supervised_not_registered");
3735 let message = if let Some(pid) = status.waiting_for_previous_pid {
3736 format!("waiting: previous instance (pid {pid}) is still running but hidden; it cannot be identified")
3737 } else {
3738 format!(
3739 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3740 status.state, status.enabled, status.live
3741 )
3742 };
3743 control_error_frame(frame, code, message)
3744 }
3745
3746 async fn handle_route_open(
3747 &self,
3748 ctx: &RouteCtx,
3749 frame: Frame,
3750 request: RouteOpenRequest,
3751 ) -> Result<Vec<Frame>, RouterError> {
3752 let RouteOpenRequest {
3753 target,
3754 mut identity,
3755 consumer_identity,
3756 consumer_capabilities,
3757 role_versions,
3758 admission_facts,
3759 scope,
3760 } = request;
3761 let target_module_id = target_module_id(&target).to_string();
3762 if self
3763 .registry
3764 .get_module_by_connection(ctx.connection_id)
3765 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3766 .is_some()
3767 {
3768 return Ok(vec![control_error_frame(
3769 &frame,
3770 "invalid_request",
3771 "module connections cannot open client routes",
3772 )?]);
3773 }
3774 debug!(
3775 connection_id = ctx.connection_id.get(),
3776 corr = frame.header.corr,
3777 module_id = %target_module_id,
3778 "handling route.open"
3779 );
3780
3781 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3787 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3788 self.observe_route_open_refusal(
3789 ctx,
3790 &target_module_id,
3791 "invalid_role_versions",
3792 error_codes::INVALID_REQUEST,
3793 );
3794 return Ok(vec![control_error_body_frame(
3795 &frame,
3796 ErrorBody {
3797 code: error_codes::INVALID_REQUEST.to_string(),
3798 message: error.to_string(),
3799 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3800 },
3801 )?]);
3802 }
3803
3804 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
3805 if status.waiting_for_previous_pid.is_some() {
3806 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3807 ctx,
3808 &frame,
3809 &target_module_id,
3810 "module_warming",
3811 &status,
3812 )?]);
3813 }
3814 }
3815
3816 let Some(registration) = self
3835 .registry
3836 .get_module(&target_module_id)
3837 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3838 else {
3839 if let Some((status, warming)) =
3840 self.supervisor_status(&target_module_id, frame.header.corr)?
3841 {
3842 if status.protocol == ModuleProtocol::None {
3849 return Ok(vec![self.route_open_refusal_frame(
3850 ctx,
3851 &frame,
3852 &target_module_id,
3853 "protocol_none",
3854 error_codes::MODULE_NO_PROTOCOL,
3855 format!(
3856 "module_id '{target_module_id}' is declared protocol: none; \
3857 it speaks no subc wire and serves no routes"
3858 ),
3859 )?]);
3860 }
3861 let code = if warming {
3862 "module_warming"
3863 } else {
3864 "target_unavailable"
3865 };
3866 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3867 ctx,
3868 &frame,
3869 &target_module_id,
3870 code,
3871 &status,
3872 )?]);
3873 }
3874 if let Some(removed_ago_ms) =
3875 self.supervisor.removal_tombstone_age_ms(&target_module_id)
3876 {
3877 return Ok(vec![self.route_open_refusal_frame(
3878 ctx,
3879 &frame,
3880 &target_module_id,
3881 "removed",
3882 error_codes::MODULE_REMOVED,
3883 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3884 )?]);
3885 }
3886 return Ok(vec![self.route_open_refusal_frame(
3887 ctx,
3888 &frame,
3889 &target_module_id,
3890 "not_registered",
3891 error_codes::UNKNOWN_MODULE,
3892 format!("module_id '{target_module_id}' is not registered"),
3893 )?]);
3894 };
3895
3896 if !registration.ready {
3900 self.counters
3901 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3902 info!(
3903 target: "control",
3904 code = error_codes::MODULE_WARMING,
3905 module_id = ?target_module_id,
3906 connection_id = ctx.connection_id.get(),
3907 reason = "declared_not_ready",
3908 "route.open refused"
3909 );
3910 self.route_outages
3913 .record_not_serving(&target_module_id, "declared_not_ready");
3914 return Ok(vec![control_error_body_frame(
3915 &frame,
3916 ErrorBody {
3917 code: error_codes::MODULE_WARMING.to_string(),
3918 message: format!(
3919 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3920 ),
3921 detail: Some(serde_json::json!({
3922 "reason": "declared_not_ready"
3923 })),
3924 },
3925 )?]);
3926 }
3927
3928 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3947 self.counters
3948 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3949 info!(
3950 target: "control",
3951 code = error_codes::MODULE_WARMING,
3952 module_id = ?target_module_id,
3953 connection_id = ctx.connection_id.get(),
3954 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3955 capability = %capability,
3956 "route.open refused"
3957 );
3958 return Ok(vec![control_error_body_frame(
3959 &frame,
3960 ErrorBody {
3961 code: error_codes::MODULE_WARMING.to_string(),
3962 message: format!(
3963 "module_id '{target_module_id}' requires capability '{capability}', \
3964 which no registered module provides; retry"
3965 ),
3966 detail: Some(serde_json::json!({
3967 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3968 "capability": capability,
3969 })),
3970 },
3971 )?]);
3972 }
3973
3974 if !target_has_required_role(&target, ®istration.manifest.provides) {
3975 return Ok(vec![self.route_open_refusal_frame(
3976 ctx,
3977 &frame,
3978 &target_module_id,
3979 "role_not_provided",
3980 "target_unavailable",
3981 format!("module_id '{target_module_id}' does not provide the requested target"),
3982 )?]);
3983 }
3984
3985 if registration.state != ChannelState::Active {
3986 return Ok(vec![self.route_open_refusal_frame(
3987 ctx,
3988 &frame,
3989 &target_module_id,
3990 "registration_not_active",
3991 "target_unavailable",
3992 format!("module_id '{target_module_id}' is not active"),
3993 )?]);
3994 }
3995
3996 if self
3997 .forwarding
3998 .module_is_draining(&target_module_id)
3999 .map_err(RouterError::Forwarding)?
4000 {
4001 return Ok(vec![self.route_open_refusal_frame(
4002 ctx,
4003 &frame,
4004 &target_module_id,
4005 "reloading",
4006 "module_reloading",
4007 format!("module_id '{target_module_id}' is reloading"),
4008 )?]);
4009 }
4010
4011 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4012 process_liveness.process_live(&target_module_id) == Some(false)
4013 }) {
4014 if process_liveness.process_replacing(&target_module_id) {
4021 return Ok(vec![self.route_open_refusal_frame(
4022 ctx,
4023 &frame,
4024 &target_module_id,
4025 "reloading",
4026 "module_reloading",
4027 format!("module_id '{target_module_id}' is reloading"),
4028 )?]);
4029 }
4030 return Ok(vec![self.route_open_refusal_frame(
4031 ctx,
4032 &frame,
4033 &target_module_id,
4034 "supervisor_not_live",
4035 "target_unavailable",
4036 format!("module_id '{target_module_id}' is not live"),
4037 )?]);
4038 }
4039
4040 if !self
4041 .forwarding
4042 .has_live_module_connection(&target_module_id)
4043 .map_err(RouterError::Forwarding)?
4044 {
4045 return Ok(vec![self.route_open_refusal_frame(
4046 ctx,
4047 &frame,
4048 &target_module_id,
4049 "no_forwarding_connection",
4050 "target_unavailable",
4051 format!("module_id '{target_module_id}' has no live forwarding connection"),
4052 )?]);
4053 }
4054
4055 if let Some(error) =
4056 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4057 {
4058 self.observe_route_open_refusal(
4059 ctx,
4060 &target_module_id,
4061 "op_not_allowed",
4062 "op_not_allowed",
4063 );
4064 return Ok(vec![error]);
4065 }
4066
4067 let principal = match self.route_open_principal(&frame, consumer_identity)? {
4068 Ok(principal) => principal,
4069 Err(error) => {
4070 self.observe_route_open_refusal(
4071 ctx,
4072 &target_module_id,
4073 "bad_consumer_identity",
4074 "bad_consumer_identity",
4075 );
4076 return Ok(vec![error]);
4077 }
4078 };
4079
4080 if let Principal::Reserved {
4084 module_id: opening_module_id,
4085 } = &principal
4086 {
4087 if let Some(opening_registration) = self
4088 .registry
4089 .get_module(opening_module_id)
4090 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4091 {
4092 if let Some(capability) =
4093 denied_capability(&opening_registration.manifest, ®istration.manifest)
4094 {
4095 warn!(
4096 opening_module_id,
4097 target_module_id,
4098 capability,
4099 "refusing route.open because an attested capability deny edge matches"
4100 );
4101 return Ok(vec![self.route_open_refusal_frame(
4102 ctx,
4103 &frame,
4104 &target_module_id,
4105 "capability_deny_edge",
4106 "capability_forbidden",
4107 format!(
4108 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4109 ),
4110 )?]);
4111 }
4112 }
4113 }
4114
4115 if admission_facts.is_some() {
4116 let carrier_matches = matches!(
4117 &principal,
4118 Principal::Reserved { module_id }
4119 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4120 );
4121 if !carrier_matches {
4122 return Ok(vec![self.route_open_refusal_frame(
4123 ctx,
4124 &frame,
4125 &target_module_id,
4126 "admission_facts_carrier_not_permitted",
4127 "admission_facts_not_permitted",
4128 "admission facts may only be carried by the configured reserved module",
4129 )?]);
4130 }
4131
4132 let target_allowed = self
4133 .admission_facts_targets
4134 .as_ref()
4135 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4136 if !target_allowed {
4137 return Ok(vec![self.route_open_refusal_frame(
4138 ctx,
4139 &frame,
4140 &target_module_id,
4141 "admission_facts_target_not_listed",
4142 "admission_facts_target_not_allowed",
4143 format!(
4144 "admission facts are not permitted for target module_id '{target_module_id}'"
4145 ),
4146 )?]);
4147 }
4148
4149 }
4153
4154 let (bound_scope, mut scope_stamp) = match scope {
4160 None => (None, None),
4161 Some(selector) => {
4162 let owner_configured = match &selector.owner {
4163 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4169 _ => false,
4170 };
4171 let admitted = self
4172 .scopes
4173 .read()
4174 .unwrap_or_else(|poisoned| poisoned.into_inner())
4175 .admit(&principal, &target_module_id, &selector, owner_configured)
4176 .and_then(|admission| {
4177 crate::scopes::check_target_flow_support(
4178 &admission.stamp,
4179 &target_module_id,
4180 registration.manifest.capabilities.as_ref(),
4181 )?;
4182 crate::scopes::check_target_agent_run_support(
4183 &admission.stamp,
4184 &target_module_id,
4185 registration.manifest.capabilities.as_ref(),
4186 )?;
4187 Ok(admission)
4188 });
4189 match admitted {
4190 Ok(admission) => (
4191 Some(BoundScope {
4192 owner: admission.owner,
4193 scope_ref: admission.stamp.scope_ref.clone(),
4194 tag: admission.tag,
4195 }),
4196 Some(admission.stamp),
4197 ),
4198 Err(refusal) => {
4199 return Ok(vec![self.route_open_refusal_frame(
4200 ctx,
4201 &frame,
4202 &target_module_id,
4203 refusal.code,
4204 refusal.code,
4205 refusal.message,
4206 )?]);
4207 }
4208 }
4209 }
4210 };
4211
4212 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4234 Ok(project_root) => project_root,
4235 Err(err) => {
4236 return Ok(vec![control_error_frame(
4237 &frame,
4238 "invalid_project_root",
4239 err.to_string(),
4240 )?])
4241 }
4242 };
4243 identity.project_root = project_root.as_path().to_path_buf();
4244
4245 let _concurrency_guard = match self
4256 .route_bind_concurrency
4257 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4258 {
4259 Ok(guard) => guard,
4260 Err(in_flight) => {
4261 return Ok(vec![self.route_open_target_capacity_refusal(
4262 ctx,
4263 &frame,
4264 &target_module_id,
4265 in_flight,
4266 )?]);
4267 }
4268 };
4269
4270 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4273 RouteBindAdmission::Admitted { guard, probe } => {
4274 if probe {
4275 info!(
4276 module_id = %target_module_id,
4277 connection_id = ctx.connection_id.get(),
4278 "route.bind breaker half-open: admitting one probe"
4279 );
4280 }
4281 guard
4282 }
4283 RouteBindAdmission::Refused {
4284 consecutive_timeouts,
4285 retry_in,
4286 probe_in_flight,
4287 } => {
4288 return Ok(vec![self.route_open_breaker_refusal_frame(
4289 ctx,
4290 &frame,
4291 &target_module_id,
4292 consecutive_timeouts,
4293 retry_in,
4294 probe_in_flight,
4295 )?]);
4296 }
4297 };
4298
4299 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4304 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4305 let pending = match self
4306 .forwarding
4307 .begin_route_bind_relay_for(
4308 ctx.connection_id,
4309 ctx.egress.clone(),
4310 response_version(&frame),
4311 frame.header.corr,
4312 &target_module_id,
4313 principal.clone(),
4314 bound_scope,
4315 Some(project_root),
4316 relay_deadline,
4317 )
4318 .await
4319 {
4320 Ok(pending) => pending,
4321 Err(err) => {
4322 return Ok(vec![self.route_open_refusal_frame(
4323 ctx,
4324 &frame,
4325 &target_module_id,
4326 "relay_reservation_failed",
4327 forwarding_error_code(&err),
4328 err.to_string(),
4329 )?])
4330 }
4331 };
4332 let crate::forwarding::PendingRouteBindRelay {
4333 endpoint,
4334 module_sink,
4335 negotiated_ver,
4336 client_channel,
4337 client_epoch,
4338 module_channel,
4339 module_epoch,
4340 corr: relay_corr,
4341 receiver,
4342 } = pending;
4343 let mut reservation =
4344 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4345
4346 let relay_registration = self
4351 .registry
4352 .get_module_by_connection(endpoint.connection_id)
4353 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4354 let relay_capabilities = relay_registration
4355 .as_ref()
4356 .and_then(|registration| registration.manifest.capabilities.as_ref());
4357 if let Some(stamp) = scope_stamp
4358 .as_ref()
4359 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4360 {
4361 if let Err(refusal) = crate::scopes::check_target_flow_support(
4362 stamp,
4363 &target_module_id,
4364 relay_capabilities,
4365 )
4366 .and_then(|()| {
4367 crate::scopes::check_target_agent_run_support(
4368 stamp,
4369 &target_module_id,
4370 relay_capabilities,
4371 )
4372 }) {
4373 reservation.release_and_disarm();
4374 return Ok(vec![self.route_open_refusal_frame(
4375 ctx,
4376 &frame,
4377 &target_module_id,
4378 refusal.code,
4379 refusal.code,
4380 refusal.message,
4381 )?]);
4382 }
4383 }
4384
4385 let call_mode = if relay_capabilities
4388 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4389 {
4390 Some(derive_call_mode(scope_stamp.as_ref()))
4391 } else {
4392 if let Some(stamp) = scope_stamp.as_mut() {
4393 stamp.expires_at_ms = None;
4394 }
4395 None
4396 };
4397
4398 debug!(
4399 connection_id = ctx.connection_id.get(),
4400 client_channel,
4401 client_epoch,
4402 module_channel,
4403 module_epoch,
4404 "reserved route handle pair"
4405 );
4406 let principal_label = principal_label(&principal);
4409 let relay = ModuleControlRequest::RouteBind {
4410 route_channel: module_channel,
4411 epoch: module_epoch,
4412 target,
4413 identity,
4414 principal: Some(principal),
4415 consumer_capabilities,
4416 role_versions,
4417 admission_facts,
4418 scope: scope_stamp,
4419 call_mode,
4420 };
4421 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4422 RouterError::backend(
4423 0,
4424 frame.header.corr,
4425 format!("failed to encode route.bind request: {err}"),
4426 )
4427 })?;
4428 let relay_frame = Frame::build_with_version(
4429 negotiated_ver,
4430 FrameType::Request,
4431 control_flags(),
4432 0,
4433 0,
4434 relay_corr,
4435 relay_body,
4436 )
4437 .map_err(RouterError::FrameBuild)?;
4438
4439 if let Err(err) = module_sink.send(relay_frame).await {
4440 reservation.release_and_disarm();
4441 return Ok(vec![self.route_open_refusal_frame(
4442 ctx,
4443 &frame,
4444 &target_module_id,
4445 "relay_send_failed",
4446 "target_unavailable",
4447 err.to_string(),
4448 )?]);
4449 }
4450
4451 #[cfg(test)]
4452 if let Some(hold) = &self.route_bind_relay_sent_hold {
4453 hold.wait().await;
4457 hold.wait().await;
4458 }
4459
4460 if self
4465 .forwarding
4466 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4467 .map_err(RouterError::Forwarding)?
4468 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4469 {
4470 self.send_abandoned_route_bind_goodbye(
4471 &module_sink,
4472 negotiated_ver,
4473 module_channel,
4474 module_epoch,
4475 );
4476 }
4477
4478 match timeout_at(relay_deadline, receiver).await {
4479 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4480 reservation.disarm();
4481 if breaker.record_accepted() {
4482 info!(
4483 module_id = %target_module_id,
4484 "route.bind breaker closed: the probe was accepted"
4485 );
4486 }
4487 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4488 Ok(Vec::new())
4489 }
4490 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4491 reservation.release_and_disarm();
4492 breaker.record_inconclusive();
4496 let scope_code = match body.code.as_str() {
4500 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4501 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4502 _ => None,
4503 };
4504 if let Some(code) = scope_code {
4505 self.observe_route_open_refusal(
4506 ctx,
4507 &target_module_id,
4508 "scope_changed_before_commit",
4509 code,
4510 );
4511 return Ok(vec![control_error_body_frame(&frame, body)?]);
4512 }
4513 self.counters
4514 .increment_route_open_refused("module_rejected");
4515 info!(
4516 target: "control",
4517 code = "module_rejected",
4518 module_code = ?body.code,
4519 module_id = ?target_module_id,
4520 connection_id = ctx.connection_id.get(),
4521 "route.open refused"
4522 );
4523 Ok(vec![control_error_body_frame(&frame, body)?])
4524 }
4525 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4526 reservation.release_and_disarm();
4527 breaker.record_inconclusive();
4528 tracing::warn!(
4535 module_id = %target_module_id,
4536 "route.bind relay abandoned: {message}"
4537 );
4538 Ok(vec![self.route_open_refusal_frame(
4539 ctx,
4540 &frame,
4541 &target_module_id,
4542 "relay_abandoned",
4543 "target_unavailable",
4544 message,
4545 )?])
4546 }
4547 Ok(Err(_)) => {
4548 reservation.release_and_disarm();
4549 breaker.record_inconclusive();
4550 Ok(vec![self.route_open_refusal_frame(
4551 ctx,
4552 &frame,
4553 &target_module_id,
4554 "relay_waiter_canceled",
4555 "target_unavailable",
4556 "route.bind relay waiter was canceled before the module responded",
4557 )?])
4558 }
4559 Err(_) => {
4560 reservation.release_and_disarm();
4561 if let Some(opened) = breaker.record_timeout(
4565 self.route_bind_breaker_threshold,
4566 self.route_bind_breaker_cooldown,
4567 ) {
4568 warn!(
4569 module_id = %target_module_id,
4570 consecutive_timeouts = opened.consecutive_timeouts,
4571 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4572 reopened_after_probe = opened.reopened_after_probe,
4573 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4574 );
4575 }
4576 tracing::warn!(
4584 module_id = %target_module_id,
4585 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4586 "route.bind relay timed out: module did not ack within budget"
4587 );
4588 Ok(vec![self.route_open_refusal_frame(
4589 ctx,
4590 &frame,
4591 &target_module_id,
4592 "relay_timed_out",
4593 "module_timeout",
4594 format!(
4595 "module_id '{target_module_id}' did not answer route.bind within {:?}",
4596 route_bind_relay_timeout
4597 ),
4598 )?])
4599 }
4600 }
4601 }
4602
4603 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4604 let response = ClientControlResponse::SupervisorSpawnSnapshot {
4605 snapshot: self.supervisor.spawn_snapshot(),
4606 };
4607 Ok(vec![control_response_body_frame(
4608 &frame,
4609 &response,
4610 "ClientControlResponse::SupervisorSpawnSnapshot",
4611 )?])
4612 }
4613
4614 fn handle_supervisor_spawn_subscribe(
4615 &self,
4616 ctx: &RouteCtx,
4617 frame: Frame,
4618 since: Option<SpawnCursor>,
4619 ) -> Result<Vec<Frame>, RouterError> {
4620 match self.supervisor.subscribe_spawns(
4621 ctx.connection_id,
4622 frame.header.corr,
4623 response_version(&frame),
4624 since,
4625 ctx.egress.clone(),
4626 ) {
4627 Ok(()) => Ok(Vec::new()),
4628 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4629 Ok(vec![control_error_body_frame(
4630 &frame,
4631 ErrorBody {
4632 code: "spawn_cursor_incarnation_mismatch".to_string(),
4633 message: "spawn cursor belongs to a different daemon incarnation"
4634 .to_string(),
4635 detail: Some(serde_json::json!({
4636 "current_daemon_incarnation": current
4637 })),
4638 },
4639 )?])
4640 }
4641 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4642 &frame,
4643 ErrorBody {
4644 code: "spawn_cursor_too_old".to_string(),
4645 message: "spawn cursor predates the retained event ring".to_string(),
4646 detail: Some(serde_json::json!({
4647 "oldest_retained_cursor": oldest
4648 })),
4649 },
4650 )?]),
4651 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4652 0,
4653 frame.header.corr,
4654 format!("failed to open supervisor spawn subscription: {error}"),
4655 )),
4656 }
4657 }
4658
4659 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4660 let generation = self
4661 .registry
4662 .generation()
4663 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4664 let mut modules = Vec::new();
4665 for module in self.supervisor.list() {
4666 let status = module.status_for_control("list").map_err(|err| {
4667 RouterError::backend(
4668 0,
4669 frame.header.corr,
4670 format!("failed to read supervisor status: {err}"),
4671 )
4672 })?;
4673 let (configured, _) = module.configuration().map_err(|err| {
4674 RouterError::backend(
4675 0,
4676 frame.header.corr,
4677 format!("failed to read module configuration: {err}"),
4678 )
4679 })?;
4680 let image = module.running_image_agreement().await;
4682 let resources = Some(module.child_resource_usage());
4685 let pending_reload = Some(reload_verdict(
4686 &configured.program,
4687 status.spawned_from.as_deref(),
4688 image,
4689 ));
4690 modules.push(
4693 SupervisorEntry::new(
4694 status.module_id,
4695 status.state.to_string(),
4696 status.enabled,
4697 status.live,
4698 status.health.status,
4699 )
4700 .with_launch_nonce_env(Some(!cfg!(unix)))
4701 .with_launch_nonce_source(status.launch_nonce_source)
4702 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
4703 .with_protocol(status.protocol)
4704 .with_pending_reload(pending_reload)
4705 .with_last_probe_ms(status.health.last_probe_ms)
4706 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
4707 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
4708 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
4709 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
4710 .with_restart_count(Some(status.restart_count))
4711 .with_max_restarts(Some(status.max_restarts))
4712 .with_lifetime_restarts(Some(status.lifetime_restarts))
4713 .with_spawn_generation(Some(status.spawn_generation))
4714 .with_restart_window_secs(Some(status.restart_window.as_secs()))
4715 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
4716 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
4717 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
4718 .with_resources(resources),
4719 );
4720 }
4721 let response = ClientControlResponse::SupervisorList {
4722 generation,
4723 modules,
4724 };
4725 Ok(vec![control_response_body_frame(
4726 &frame,
4727 &response,
4728 "ClientControlResponse::SupervisorList",
4729 )?])
4730 }
4731
4732 fn handle_supervisor_stderr_tail(
4733 &self,
4734 frame: Frame,
4735 module_id: String,
4736 max_lines: Option<u32>,
4737 max_bytes: Option<u32>,
4738 ) -> Result<Vec<Frame>, RouterError> {
4739 let Some(module) = self.supervisor.get(&module_id) else {
4740 return Ok(vec![control_error_frame(
4741 &frame,
4742 "unknown_module",
4743 format!("module_id '{module_id}' is not supervised"),
4744 )?]);
4745 };
4746
4747 let snapshot = module.stderr_tail(
4748 max_lines.map(|value| value as usize),
4749 max_bytes.map(|value| value as usize),
4750 );
4751
4752 let response = ClientControlResponse::SupervisorStderrTail {
4753 module_id,
4754 tail: StderrTail {
4755 capture: match snapshot.capture {
4756 CaptureState::Captured => StderrCaptureState::Captured,
4757 CaptureState::Incomplete { reason } => {
4758 StderrCaptureState::Incomplete { reason }
4759 }
4760 CaptureState::NotCaptured { reason } => {
4761 StderrCaptureState::NotCaptured { reason }
4762 }
4763 },
4764 entries: snapshot
4765 .entries
4766 .into_iter()
4767 .map(|entry| match entry {
4768 TailEntry::Line {
4769 text,
4770 truncated,
4771 at_ms,
4772 } => StderrTailEntry::Line {
4773 text,
4774 truncated,
4775 at_ms,
4776 },
4777 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
4778 })
4779 .collect(),
4780 dropped_lines: snapshot.dropped_lines,
4781 },
4782 };
4783 Ok(vec![control_response_body_frame(
4784 &frame,
4785 &response,
4786 "ClientControlResponse::SupervisorStderrTail",
4787 )?])
4788 }
4789
4790 async fn handle_supervisor_terminals(
4791 &self,
4792 frame: Frame,
4793 module_id: String,
4794 ) -> Result<Vec<Frame>, RouterError> {
4795 let Some(module) = self.supervisor.get(&module_id) else {
4796 return Ok(vec![control_error_frame(
4797 &frame,
4798 "unknown_module",
4799 format!("module_id '{module_id}' is not supervised"),
4800 )?]);
4801 };
4802
4803 let terminals = module
4806 .read_durable_terminal_history()
4807 .await
4808 .map_err(|error| {
4809 RouterError::backend(
4810 0,
4811 frame.header.corr,
4812 format!("failed to read terminal history: {error}"),
4813 )
4814 })?;
4815 let response = ClientControlResponse::SupervisorTerminals {
4816 module_id,
4817 terminals,
4818 };
4819 Ok(vec![control_response_body_frame(
4820 &frame,
4821 &response,
4822 "ClientControlResponse::SupervisorTerminals",
4823 )?])
4824 }
4825
4826 fn handle_supervisor_routes(
4827 &self,
4828 frame: Frame,
4829 module_id: Option<String>,
4830 ) -> Result<Vec<Frame>, RouterError> {
4831 let modules = self
4832 .forwarding
4833 .route_census(module_id.as_deref())
4834 .map_err(RouterError::Forwarding)?
4835 .into_iter()
4836 .map(|(module_id, routes)| SupervisorRouteModule {
4837 module_id,
4838 routes: routes
4839 .into_iter()
4840 .map(|route| SupervisorRoute {
4841 consumer: match route.principal {
4842 Principal::Reserved { module_id } => {
4843 SupervisorRouteConsumer::Reserved { module_id }
4844 }
4845 Principal::Direct | Principal::Unverified => {
4846 SupervisorRouteConsumer::Direct {
4847 connection_id: route.goodbye_target.connection_id.get(),
4848 }
4849 }
4850 },
4851 age_ms: Instant::now()
4852 .saturating_duration_since(route.bound_at)
4853 .as_millis()
4854 .try_into()
4855 .unwrap_or(u64::MAX),
4856 draining: route.draining,
4857 drain_reason: route.drain_reason,
4858 })
4859 .collect(),
4860 })
4861 .collect();
4862 let response = ClientControlResponse::SupervisorRoutes { modules };
4863 Ok(vec![control_response_body_frame(
4864 &frame,
4865 &response,
4866 "ClientControlResponse::SupervisorRoutes",
4867 )?])
4868 }
4869
4870 async fn handle_supervisor_provenance(
4871 &self,
4872 frame: Frame,
4873 module_id: Option<String>,
4874 ) -> Result<Vec<Frame>, RouterError> {
4875 let mut selected = if let Some(module_id) = module_id {
4876 let Some(module) = self.supervisor.get(&module_id) else {
4877 return Ok(vec![control_error_frame(
4878 &frame,
4879 "unknown_module",
4880 format!("module_id '{module_id}' is not supervised"),
4881 )?]);
4882 };
4883 vec![module]
4884 } else {
4885 self.supervisor.list()
4886 };
4887
4888 let mut modules = Vec::with_capacity(selected.len());
4889 for module in selected.drain(..) {
4890 let (status, observed_image) = module
4891 .status_and_running_image_agreement()
4892 .await
4893 .map_err(|err| {
4894 RouterError::backend(
4895 0,
4896 frame.header.corr,
4897 format!("failed to read supervisor status: {err}"),
4898 )
4899 })?;
4900 let module_declared = self
4901 .registry
4902 .get_module(&status.module_id)
4903 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4904 .and_then(|registration| registration.manifest.provenance)
4905 .map(|build| ModuleDeclaredProvenance::Reported { build })
4906 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4907 #[cfg(test)]
4908 let running_image = match &self.provenance_probe_override {
4909 Some(result) => result.clone(),
4910 None => observed_image,
4911 };
4912 #[cfg(not(test))]
4913 let running_image = observed_image;
4914 modules.push(SupervisorModuleProvenance {
4915 module_id: status.module_id,
4916 module_declared,
4917 daemon_observed: SupervisorObservedProcess {
4918 launch_nonce_source: status.launch_nonce_source,
4919 pid: status.pid,
4920 spawned_at_ms: status.spawned_at_ms,
4921 spawned_from: status.spawned_from,
4922 running_image,
4923 },
4924 });
4925 }
4926 let daemon = SupervisorDaemonProvenance {
4927 daemon_build: self.daemon_provenance.build.clone(),
4928 daemon_observed: DaemonObservedProcess {
4929 pid: self.daemon_provenance.pid,
4930 started_at_ms: self
4931 .daemon_provenance
4932 .start_clock
4933 .map(|clock| clock.started_at_ms())
4934 .or(self.daemon_provenance.started_at_ms),
4935 running_image: self
4936 .daemon_provenance
4937 .probe
4938 .observe(
4939 self.daemon_provenance.pid,
4940 self.daemon_provenance.executable_path.as_deref(),
4941 self.daemon_provenance.executable_identity.clone(),
4942 self.daemon_provenance.process_start_time,
4943 )
4944 .await,
4945 },
4946 };
4947 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
4948 Ok(vec![control_response_body_frame(
4949 &frame,
4950 &response,
4951 "ClientControlResponse::SupervisorProvenance",
4952 )?])
4953 }
4954
4955 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4956 self.refresh_capability_requirements();
4957 let generation = self
4958 .registry
4959 .generation()
4960 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4961 let modules = self
4962 .supervisor
4963 .list()
4964 .into_iter()
4965 .map(|module| {
4966 let status = module.status_for_control("health").map_err(|err| {
4967 RouterError::backend(
4968 0,
4969 frame.header.corr,
4970 format!("failed to read supervisor health: {err}"),
4971 )
4972 })?;
4973 let module_id = status.module_id;
4974 let capability_detail = self
4975 .capability_evaluator
4976 .required_problem_detail(&module_id);
4977 Ok(SupervisorHealthEntry {
4978 module_id,
4979 status: status.health.status,
4980 detail: append_capability_problem_detail(
4981 status.health.detail,
4982 capability_detail,
4983 ),
4984 metrics: status.health.metrics,
4985 consecutive_failures: status.health.consecutive_failures,
4986 late_answer_count: status.health.late_answer_count,
4987 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4988 last_action: status.health.last_action,
4989 last_action_ms: status.health.last_action_ms,
4990 last_probe_ms: status.health.last_probe_ms,
4991 })
4992 })
4993 .collect::<Result<Vec<_>, RouterError>>()?;
4994 let response = ClientControlResponse::SupervisorHealth {
4995 generation,
4996 modules,
4997 };
4998 Ok(vec![control_response_body_frame(
4999 &frame,
5000 &response,
5001 "ClientControlResponse::SupervisorHealth",
5002 )?])
5003 }
5004
5005 async fn handle_supervisor_restart(
5006 &self,
5007 frame: Frame,
5008 module_id: String,
5009 drain_timeout_ms: Option<u64>,
5010 requester: Principal,
5011 ) -> Result<Vec<Frame>, RouterError> {
5012 let operation_lock = self.supervisor.operation_lock();
5013 let _operation_guard = operation_lock.lock().await;
5014 let Some(module) = self.supervisor.get(&module_id) else {
5015 return Ok(vec![control_error_frame(
5016 &frame,
5017 "unknown_module",
5018 format!("module_id '{module_id}' is not supervised"),
5019 )?]);
5020 };
5021
5022 self.route_outages.mark_operator_action(&module_id);
5023 if let Err(err) = module
5024 .restart_requested(drain_timeout_ms, Some(requester))
5025 .await
5026 {
5027 self.route_outages
5028 .operator_action_ended_unrefused(&module_id);
5029 let (code, message) = match err {
5030 crate::supervise::SuperviseError::Disabled { .. } => {
5031 ("module_disabled", err.to_string())
5032 }
5033 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5034 ("swap_in_progress", err.to_string())
5035 }
5036 _ => (
5037 "target_unavailable",
5038 format!("failed to restart module_id '{module_id}': {err}"),
5039 ),
5040 };
5041 return Ok(vec![control_error_frame(&frame, code, message)?]);
5042 }
5043
5044 let response = ClientControlResponse::SupervisorAck {
5045 module_id,
5046 applied: true,
5047 };
5048 Ok(vec![control_response_body_frame(
5049 &frame,
5050 &response,
5051 "ClientControlResponse::SupervisorAck",
5052 )?])
5053 }
5054
5055 async fn handle_supervisor_swap(
5059 &self,
5060 frame: Frame,
5061 module_id: String,
5062 ready_timeout_ms: Option<u64>,
5063 requester: Principal,
5064 ) -> Result<Vec<Frame>, RouterError> {
5065 let module = {
5072 let operation_lock = self.supervisor.operation_lock();
5073 let _operation_guard = operation_lock.lock().await;
5074 self.supervisor.get(&module_id)
5075 };
5076 let Some(module) = module else {
5077 return Ok(vec![control_error_frame(
5078 &frame,
5079 "unknown_module",
5080 format!("module_id '{module_id}' is not supervised"),
5081 )?]);
5082 };
5083
5084 self.route_outages.mark_operator_action(&module_id);
5085 if let Err(err) = module
5086 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5087 .await
5088 {
5089 self.route_outages
5090 .operator_action_ended_unrefused(&module_id);
5091 use crate::supervise::SuperviseError;
5092 let message = err.to_string();
5093 let error = match err {
5094 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5095 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5096 code: "swap_refused".to_string(),
5097 message,
5098 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5099 },
5100 SuperviseError::SwapFailed {
5101 arm,
5102 candidate_exit,
5103 ..
5104 } => ErrorBody {
5105 code: "swap_failed".to_string(),
5106 message,
5107 detail: Some(serde_json::json!({
5108 "arm": arm.as_str(),
5109 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5110 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5111 })),
5112 },
5113 _ => ErrorBody::new(
5114 "target_unavailable",
5115 format!("failed to swap module_id '{module_id}': {message}"),
5116 ),
5117 };
5118 return Ok(vec![control_error_body_frame(&frame, error)?]);
5119 }
5120 self.route_outages
5124 .operator_action_ended_unrefused(&module_id);
5125
5126 let response = ClientControlResponse::SupervisorAck {
5127 module_id,
5128 applied: true,
5129 };
5130 Ok(vec![control_response_body_frame(
5131 &frame,
5132 &response,
5133 "ClientControlResponse::SupervisorAck",
5134 )?])
5135 }
5136
5137 async fn handle_supervisor_reload(
5138 &self,
5139 frame: Frame,
5140 module_id: String,
5141 requester: Principal,
5142 ) -> Result<Vec<Frame>, RouterError> {
5143 let operation_lock = self.supervisor.operation_lock();
5144 let _operation_guard = operation_lock.lock().await;
5145 let Some(module) = self.supervisor.get(&module_id) else {
5146 return Ok(vec![control_error_frame(
5147 &frame,
5148 "unknown_module",
5149 format!("module_id '{module_id}' is not supervised"),
5150 )?]);
5151 };
5152
5153 self.route_outages.mark_operator_action(&module_id);
5154 if let Err(err) = module.reload_requested(Some(requester)).await {
5155 self.route_outages
5156 .operator_action_ended_unrefused(&module_id);
5157 let (code, message) = match err {
5158 crate::supervise::SuperviseError::Disabled { .. } => {
5159 ("module_disabled", err.to_string())
5160 }
5161 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5162 ("swap_in_progress", err.to_string())
5163 }
5164 _ => (
5165 "reload_failed",
5166 format!("failed to reload module_id '{module_id}': {err}"),
5167 ),
5168 };
5169 return Ok(vec![control_error_frame(&frame, code, message)?]);
5170 }
5171
5172 let response = ClientControlResponse::SupervisorAck {
5173 module_id,
5174 applied: true,
5175 };
5176 Ok(vec![control_response_body_frame(
5177 &frame,
5178 &response,
5179 "ClientControlResponse::SupervisorAck",
5180 )?])
5181 }
5182
5183 async fn handle_supervisor_rescan(
5184 &self,
5185 frame: Frame,
5186 preview: bool,
5187 requester: Principal,
5188 ) -> Result<Vec<Frame>, RouterError> {
5189 let Some(context) = self.rescan.clone() else {
5190 return Ok(vec![control_error_frame(
5191 &frame,
5192 "rescan_unavailable",
5193 "the daemon was not started with a reloadable config path".to_string(),
5194 )?]);
5195 };
5196
5197 let operation_lock = self.supervisor.operation_lock();
5198 let _operation_guard = operation_lock.lock().await;
5199 let loaded = match crate::daemon_config::load_with_activity(&context.config_path) {
5200 Ok(config) => config,
5201 Err(err) => {
5202 return Ok(vec![control_error_frame(
5203 &frame,
5204 "invalid_daemon_config",
5205 format!("supervisor rescan rejected daemon config: {err}"),
5206 )?])
5207 }
5208 };
5209 let Some((config, activity_config)) = loaded else {
5217 return Ok(vec![control_error_frame(
5218 &frame,
5219 "invalid_daemon_config",
5220 format!(
5221 "daemon config not found at {}; refusing to rescan (an absent config would \
5222 retire every supervised module)",
5223 context.config_path.display()
5224 ),
5225 )?]);
5226 };
5227 let (
5228 configured_port,
5229 storage_config,
5230 admission_facts_carrier_module_id,
5231 admission_facts_targets,
5232 scope_authority_owners,
5233 modules,
5234 reserved_capabilities,
5235 ) = (
5236 config.port,
5237 config.storage,
5238 config.admission_facts_carrier_module_id,
5239 config.admission_facts_targets,
5240 config.scope_authority_owners,
5241 config.modules,
5242 config.reserved_capabilities,
5243 );
5244
5245 let mut restart_required = Vec::new();
5254 for section in RestartRequiredSection::ALL {
5255 let changed = match section {
5256 RestartRequiredSection::Port => configured_port != context.configured_port,
5257 RestartRequiredSection::Storage => storage_config != context.storage_config,
5258 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5259 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5260 }
5261 RestartRequiredSection::AdmissionFactsTargets => {
5262 admission_facts_targets != context.admission_facts_targets
5263 }
5264 RestartRequiredSection::ScopeAuthorityOwners => {
5265 scope_authority_owners != context.scope_authority_owners
5266 }
5267 };
5268 if changed {
5269 restart_required.push(section.label().to_string());
5270 }
5271 }
5272 if !restart_required.is_empty() {
5273 warn!(
5274 config_path = %context.config_path.display(),
5275 sections = %restart_required.join(", "),
5276 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5277 );
5278 }
5279
5280 for configured in &modules {
5281 if let Err(err) = validate_spec(&configured.module_spec()) {
5282 return Ok(vec![control_error_frame(
5283 &frame,
5284 "invalid_daemon_config",
5285 format!("supervisor rescan rejected daemon config: {err}"),
5286 )?]);
5287 }
5288 }
5289
5290 let configured_capabilities = modules
5291 .iter()
5292 .map(|module| (module.module_id.clone(), module.enabled))
5293 .collect::<Vec<_>>();
5294 let preview_capability_warnings = if preview {
5295 let (_, registrations) = self.runtime_capability_snapshot()?;
5296 let current_modules = self
5297 .supervisor
5298 .list()
5299 .into_iter()
5300 .map(|module| module.module_id().to_string())
5301 .collect::<BTreeSet<_>>();
5302 let resulting_modules = configured_capabilities.clone();
5303 let removed = current_modules
5304 .into_iter()
5305 .filter(|module_id| {
5306 !resulting_modules
5307 .iter()
5308 .any(|(configured_id, _)| configured_id == module_id)
5309 })
5310 .collect::<Vec<_>>();
5311 self.capability_evaluator.preview_removal_warnings(
5312 resulting_modules,
5313 &removed,
5314 ®istrations,
5315 )
5316 } else {
5317 Vec::new()
5318 };
5319 let result = match self
5320 .reconcile_supervised_modules(&context.supervisor, modules, preview, requester.clone())
5321 .await
5322 {
5323 Ok(result) => result,
5324 Err(message) => {
5325 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5326 }
5327 };
5328 if !preview {
5329 if let Some(ledger) = &self.activity {
5330 ledger.module_rescanned(&result.added, &result.removed, requester);
5331 }
5332 if let Err(code) = self.configure_activity(activity_config).await {
5333 warn!(
5334 code,
5335 "activity retention could not be applied during rescan"
5336 );
5337 }
5338 self.capability_evaluator
5339 .configure(configured_capabilities, reserved_capabilities);
5340 self.capability_evaluator.wake_deadline_loop();
5341 self.refresh_capability_requirements();
5342 }
5343 let mut result = result;
5344 result.restart_required = restart_required;
5345 result.capability_warnings = preview_capability_warnings;
5346 let response = ClientControlResponse::SupervisorRescan { result };
5347 Ok(vec![control_response_body_frame(
5348 &frame,
5349 &response,
5350 "ClientControlResponse::SupervisorRescan",
5351 )?])
5352 }
5353
5354 async fn handle_supervisor_release_reserved(
5355 &self,
5356 frame: Frame,
5357 module_id: String,
5358 ) -> Result<Vec<Frame>, RouterError> {
5359 let Some(context) = self.rescan.clone() else {
5360 return Ok(vec![control_error_frame(
5361 &frame,
5362 "release_unavailable",
5363 "reserved-id release requires a daemon started with a reloadable config path",
5364 )?]);
5365 };
5366 let operation_lock = self.supervisor.operation_lock();
5367 let _operation_guard = operation_lock.lock().await;
5368 let loaded = match crate::daemon_config::load(&context.config_path) {
5369 Ok(Some(config)) => config,
5370 Ok(None) => {
5371 return Ok(vec![control_error_frame(
5372 &frame,
5373 "invalid_daemon_config",
5374 format!(
5375 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5376 context.config_path.display()
5377 ),
5378 )?])
5379 }
5380 Err(err) => {
5381 return Ok(vec![control_error_frame(
5382 &frame,
5383 "invalid_daemon_config",
5384 format!("unable to verify reserved-id release against daemon config: {err}"),
5385 )?])
5386 }
5387 };
5388 if loaded
5389 .modules
5390 .iter()
5391 .any(|configured| configured.module_id == module_id)
5392 {
5393 return Ok(vec![control_error_frame(
5394 &frame,
5395 "reserved_module_configured",
5396 format!(
5397 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5398 ),
5399 )?]);
5400 }
5401 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5402 return Ok(vec![control_error_frame(
5403 &frame,
5404 "reserved_gate_not_retained",
5405 format!(
5406 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5407 ),
5408 )?]);
5409 }
5410
5411 let response = ClientControlResponse::SupervisorAck {
5412 module_id,
5413 applied: true,
5414 };
5415 Ok(vec![control_response_body_frame(
5416 &frame,
5417 &response,
5418 "ClientControlResponse::SupervisorAck",
5419 )?])
5420 }
5421
5422 async fn reconcile_supervised_modules(
5431 &self,
5432 supervisor: &Supervisor,
5433 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5434 preview: bool,
5435 requester: Principal,
5436 ) -> Result<SupervisorRescanResult, String> {
5437 let mut current = BTreeMap::new();
5438 for module in self.supervisor.list() {
5439 let (spec, health) = module.configuration().map_err(|err| {
5440 format!(
5441 "failed to read configuration for module_id '{}': {err}",
5442 module.module_id()
5443 )
5444 })?;
5445 let enabled = module
5446 .status()
5447 .map_err(|err| {
5448 format!(
5449 "failed to read status for module_id '{}': {err}",
5450 module.module_id()
5451 )
5452 })?
5453 .enabled;
5454 current.insert(
5455 module.module_id().to_string(),
5456 (module, spec, health, enabled),
5457 );
5458 }
5459 let configured = configured_modules
5460 .into_iter()
5461 .map(|module| (module.module_id.clone(), module))
5462 .collect::<BTreeMap<_, _>>();
5463
5464 let added = configured
5465 .keys()
5466 .filter(|module_id| !current.contains_key(*module_id))
5467 .cloned()
5468 .collect::<Vec<_>>();
5469 let removed = current
5470 .keys()
5471 .filter(|module_id| !configured.contains_key(*module_id))
5472 .cloned()
5473 .collect::<Vec<_>>();
5474 let mut changed_pending_reload = Vec::new();
5475 let mut configuration_changes = BTreeSet::new();
5476 let mut enabled_changes = BTreeSet::new();
5477 let mut unchanged = 0_u32;
5478
5479 for (module_id, configured_module) in &configured {
5480 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5481 else {
5482 continue;
5483 };
5484 let launch_changed = *current_spec != configured_module.module_spec();
5488 let configuration_changed =
5489 launch_changed || *current_health != configured_module.health;
5490 let enabled_changed = *current_enabled != configured_module.enabled;
5491 if configuration_changed {
5492 configuration_changes.insert(module_id.clone());
5493 }
5494 if launch_changed {
5495 changed_pending_reload.push(module_id.clone());
5496 }
5497 if enabled_changed {
5498 enabled_changes.insert(module_id.clone());
5499 }
5500 if !configuration_changed && !enabled_changed {
5501 unchanged = unchanged.saturating_add(1);
5502 }
5503 }
5504
5505 if preview {
5511 return Ok(SupervisorRescanResult {
5512 added,
5513 removed,
5514 changed_pending_reload,
5515 enabled_changes: enabled_changes.iter().cloned().collect(),
5516 unchanged,
5517 preview: true,
5518 restart_required: Vec::new(),
5522 capability_warnings: Vec::new(),
5523 });
5524 }
5525
5526 for module_id in &removed {
5527 let module = ¤t
5528 .get(module_id)
5529 .expect("removed module came from current supervisor state")
5530 .0;
5531 module
5532 .retire_requested(Some(requester.clone()))
5533 .await
5534 .map_err(|err| {
5535 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5536 })?;
5537 self.supervisor.record_rescan_removal(module_id);
5571 self.supervisor.retire(module_id);
5572 self.route_outages.forget(module_id);
5573 }
5574
5575 for module_id in configured.keys() {
5576 let Some((module, _, _, _)) = current.get(module_id) else {
5577 continue;
5578 };
5579 let configured_module = configured
5580 .get(module_id)
5581 .expect("configured module id came from configured map");
5582 if configuration_changes.contains(module_id) {
5583 module
5584 .update_configuration(
5585 configured_module.module_spec(),
5586 configured_module.health.clone(),
5587 configured_module.drain_timeout_ms,
5588 )
5589 .await
5590 .map_err(|err| {
5591 format!(
5592 "failed to update module_id '{module_id}' configuration during rescan: {err}"
5593 )
5594 })?;
5595 }
5596 if enabled_changes.contains(module_id) {
5597 self.route_outages.mark_operator_action(module_id);
5600 module
5601 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
5602 .await
5603 .map_err(|err| {
5604 self.route_outages.operator_action_ended_unrefused(module_id);
5605 format!(
5606 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5607 configured_module.enabled
5608 )
5609 })?;
5610 }
5611 }
5612
5613 for module_id in &added {
5614 let configured_module = configured
5615 .get(module_id)
5616 .expect("added module id came from configured map");
5617 supervisor
5618 .supervise_configured_requested(
5619 configured_module.module_spec(),
5620 configured_module.enabled,
5621 configured_module.health.clone(),
5622 configured_module.drain_timeout_ms,
5623 configured_module.restart,
5624 Some(requester.clone()),
5625 )
5626 .map_err(|err| {
5627 format!("failed to add module_id '{module_id}' during rescan: {err}")
5628 })?;
5629 }
5630
5631 Ok(SupervisorRescanResult {
5632 added,
5633 removed,
5634 changed_pending_reload,
5635 enabled_changes: enabled_changes.iter().cloned().collect(),
5636 unchanged,
5637 preview: false,
5638 restart_required: Vec::new(),
5641 capability_warnings: Vec::new(),
5642 })
5643 }
5644
5645 async fn handle_supervisor_set_enabled(
5646 &self,
5647 frame: Frame,
5648 module_id: String,
5649 enabled: bool,
5650 requester: Principal,
5651 ) -> Result<Vec<Frame>, RouterError> {
5652 let operation_lock = self.supervisor.operation_lock();
5653 let _operation_guard = operation_lock.lock().await;
5654 let Some(module) = self.supervisor.get(&module_id) else {
5655 return Ok(vec![control_error_frame(
5656 &frame,
5657 "unknown_module",
5658 format!("module_id '{module_id}' is not supervised"),
5659 )?]);
5660 };
5661
5662 self.route_outages.mark_operator_action(&module_id);
5665 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
5666 Ok(applied) => applied,
5667 Err(err) => {
5668 self.route_outages
5669 .operator_action_ended_unrefused(&module_id);
5670 return Ok(vec![control_error_frame(
5671 &frame,
5672 "target_unavailable",
5673 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
5674 )?]);
5675 }
5676 };
5677 if !applied {
5678 self.route_outages
5681 .operator_action_ended_unrefused(&module_id);
5682 }
5683
5684 self.capability_evaluator.wake_deadline_loop();
5685 self.refresh_capability_requirements();
5686 let response = ClientControlResponse::SupervisorAck { module_id, applied };
5687 Ok(vec![control_response_body_frame(
5688 &frame,
5689 &response,
5690 "ClientControlResponse::SupervisorAck",
5691 )?])
5692 }
5693
5694 async fn handle_supervisor_health_probe(
5695 &self,
5696 frame: Frame,
5697 module_id: String,
5698 ) -> Result<Vec<Frame>, RouterError> {
5699 self.refresh_capability_requirements();
5700 let Some(registration) = self
5701 .registry
5702 .get_module(&module_id)
5703 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5704 else {
5705 return Ok(vec![control_error_frame(
5706 &frame,
5707 "unknown_module",
5708 format!("module_id '{module_id}' is not registered"),
5709 )?]);
5710 };
5711
5712 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
5725 {
5726 return Ok(vec![control_error_frame(
5727 &frame,
5728 "health_not_advertised",
5729 format!("module_id '{module_id}' did not advertise health.check"),
5730 )?]);
5731 }
5732
5733 let deadline = Instant::now() + self.health_probe_timeout;
5734 let pending = match self.forwarding.begin_module_control_rpc_for(
5735 &module_id,
5736 MODULE_CONTROL_OP_HEALTH_CHECK,
5737 deadline,
5738 ) {
5739 Ok(pending) => pending,
5740 Err(err) => {
5741 return Ok(vec![control_error_frame(
5742 &frame,
5743 forwarding_error_code(&err),
5744 err.to_string(),
5745 )?])
5746 }
5747 };
5748
5749 let PendingModuleControlRpc {
5750 endpoint,
5751 module_sink,
5752 negotiated_ver,
5753 corr: probe_corr,
5754 receiver,
5755 } = pending;
5756 let mut guard =
5757 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
5758 let probe_body =
5759 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5760 RouterError::backend(
5761 0,
5762 frame.header.corr,
5763 format!("failed to encode health.check request: {err}"),
5764 )
5765 })?;
5766 let probe_frame = Frame::build_with_version(
5767 negotiated_ver,
5768 FrameType::Request,
5769 control_flags(),
5770 0,
5771 0,
5772 probe_corr,
5773 probe_body,
5774 )
5775 .map_err(RouterError::FrameBuild)?;
5776
5777 if let Err(err) = module_sink.send(probe_frame).await {
5778 return Ok(vec![control_error_frame(
5779 &frame,
5780 "target_unavailable",
5781 err.to_string(),
5782 )?]);
5783 }
5784
5785 match timeout_at(deadline, receiver).await {
5786 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5787 guard.disarm();
5788 let Some(report) = response.health_report() else {
5789 return Ok(vec![control_error_frame(
5790 &frame,
5791 "invalid_control_body",
5792 "health.check RPC returned a non-health response",
5793 )?]);
5794 };
5795 let HealthReport {
5800 status,
5801 detail,
5802 metrics,
5803 } = report;
5804 let capability_detail = self
5805 .capability_evaluator
5806 .required_problem_detail(&module_id);
5807 let response = ClientControlResponse::SupervisorHealthProbe {
5808 module_id,
5809 status,
5810 detail: append_capability_problem_detail(detail, capability_detail),
5811 metrics,
5812 };
5813 Ok(vec![control_response_body_frame(
5814 &frame,
5815 &response,
5816 "ClientControlResponse::SupervisorHealthProbe",
5817 )?])
5818 }
5819 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
5820 guard.disarm();
5821 Ok(vec![control_error_body_frame(&frame, body)?])
5822 }
5823 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5824 guard.disarm();
5825 Ok(vec![control_error_frame(
5826 &frame,
5827 "target_unavailable",
5828 message,
5829 )?])
5830 }
5831 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5832 guard.disarm();
5833 Ok(vec![control_error_frame(
5834 &frame,
5835 "invalid_control_body",
5836 message,
5837 )?])
5838 }
5839 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5840 guard.disarm();
5841 Ok(vec![control_error_frame(
5842 &frame,
5843 "invalid_control_body",
5844 format!("expected module-control op '{expected}', got '{actual}'"),
5845 )?])
5846 }
5847 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
5848 guard.disarm();
5849 Ok(vec![control_error_frame(
5850 &frame,
5851 "module_timeout",
5852 format!(
5853 "module_id '{module_id}' answered health.check after {:?}",
5854 self.health_probe_timeout
5855 ),
5856 )?])
5857 }
5858 Ok(Err(_)) => Ok(vec![control_error_frame(
5859 &frame,
5860 "target_unavailable",
5861 "health.check waiter was canceled before the module responded",
5862 )?]),
5863 Err(_) => Ok(vec![control_error_frame(
5864 &frame,
5865 "module_timeout",
5866 format!(
5867 "module_id '{module_id}' did not answer health.check within {:?}",
5868 self.health_probe_timeout
5869 ),
5870 )?]),
5871 }
5872 }
5873
5874 fn supervisor_status(
5875 &self,
5876 module_id: &str,
5877 corr: u64,
5878 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5879 self.supervisor
5880 .get(module_id)
5881 .map(|module| {
5882 let warming = module.is_warming_for_control("status").map_err(|err| {
5883 RouterError::backend(
5884 0,
5885 corr,
5886 format!(
5887 "failed to read supervisor warming state for module_id '{module_id}': {err}"
5888 ),
5889 )
5890 })?;
5891 module.status_for_control("status").map_err(|err| {
5892 RouterError::backend(
5893 0,
5894 corr,
5895 format!(
5896 "failed to read supervisor status for module_id '{module_id}': {err}"
5897 ),
5898 )
5899 }).map(|status| (status, warming))
5900 })
5901 .transpose()
5902 }
5903
5904 fn guard_module_control_op(
5905 &self,
5906 frame: &Frame,
5907 module_id: &str,
5908 op: &str,
5909 ) -> Result<Option<Frame>, RouterError> {
5910 if self.module_grants_op(module_id, op, frame.header.corr)? {
5911 return Ok(None);
5912 }
5913
5914 Ok(Some(control_error_frame(
5915 frame,
5916 "op_not_allowed",
5917 format!("module_id '{module_id}' did not grant control op '{op}'"),
5918 )?))
5919 }
5920
5921 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
5922 let Some(registration) = self
5923 .registry
5924 .get_module(module_id)
5925 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
5926 else {
5927 return Ok(false);
5928 };
5929 Ok(module_registration_grants_op(®istration.control_ops, op))
5930 }
5931
5932 fn handle_status_update(
5933 &self,
5934 endpoint: ModuleEndpointId,
5935 frame: Frame,
5936 ) -> Result<Vec<Frame>, RouterError> {
5937 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
5938 Ok(update) => update,
5939 Err(err) => {
5940 if is_known_module_push_op(&frame.body) {
5945 return Ok(vec![control_error_frame(
5946 &frame,
5947 "invalid_control_body",
5948 format!("malformed module control push body: {err}"),
5949 )?]);
5950 }
5951 return Ok(Vec::new());
5952 }
5953 };
5954
5955 match update {
5956 ModuleControlPush::RouteStatus {
5957 route_channel,
5958 route_epoch,
5959 status,
5960 } => {
5961 self.forwarding
5962 .cache_status(endpoint, route_channel, route_epoch, status)
5963 .map_err(RouterError::Forwarding)?;
5964 }
5965 }
5966 Ok(Vec::new())
5967 }
5968
5969 fn handle_route_poll(
5970 &self,
5971 ctx: &RouteCtx,
5972 frame: Frame,
5973 route_channel: u16,
5974 route_epoch: u32,
5975 kind: PollKind,
5976 ) -> Result<Vec<Frame>, RouterError> {
5977 let snapshot = self
5978 .forwarding
5979 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
5980 .map_err(RouterError::Forwarding)?;
5981 let response = match (kind, snapshot) {
5982 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
5983 ClientControlResponse::RoutePoll {
5984 route_channel,
5985 route_epoch,
5986 status,
5987 live: None,
5988 }
5989 }
5990 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5991 route_channel,
5992 route_epoch,
5993 status: None,
5994 live: None,
5995 },
5996 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5997 let live = self
6023 .process_liveness
6024 .as_ref()
6025 .and_then(|source| source.process_live(&module_id))
6026 .unwrap_or(true);
6027 ClientControlResponse::RoutePoll {
6028 route_channel,
6029 route_epoch,
6030 status: None,
6031 live: Some(live),
6032 }
6033 }
6034 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6035 route_channel,
6036 route_epoch,
6037 status: None,
6038 live: Some(false),
6039 },
6040 };
6041
6042 Ok(vec![control_response_body_frame(
6043 &frame,
6044 &response,
6045 "ClientControlResponse::RoutePoll",
6046 )?])
6047 }
6048
6049 pub(crate) fn observe_module_control_completion(
6050 &self,
6051 completion: ModuleControlRpcCompletion,
6052 ) -> bool {
6053 match completion {
6054 ModuleControlRpcCompletion::Unknown => false,
6055 ModuleControlRpcCompletion::Settled => true,
6056 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6057 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6058 info!(
6059 module_id = %module_id,
6060 latency_ms,
6061 "late health.check answer proves the module is alive"
6062 );
6063 match self
6064 .supervisor
6065 .record_late_health_answer(&module_id, latency_ms)
6066 {
6067 Ok(true) => {}
6068 Ok(false) => debug!(
6069 module_id = %module_id,
6070 latency_ms,
6071 "late health.check answer has no active supervisor snapshot"
6072 ),
6073 Err(err) => warn!(
6074 module_id = %module_id,
6075 latency_ms,
6076 error = %err,
6077 "failed to record late health.check answer"
6078 ),
6079 }
6080 true
6081 }
6082 }
6083 }
6084
6085 fn refuse_to_end_module_connection_for_a_client(
6106 &self,
6107 module_connection_id: ConnectionId,
6108 corr: u64,
6109 err: ForwardingError,
6110 ) -> Result<(), RouterError> {
6111 if let ForwardingError::ConnectionClosing { connection_id } = err {
6112 if connection_id != module_connection_id {
6113 warn!(
6114 module_connection_id = module_connection_id.get(),
6115 client_connection_id = connection_id.get(),
6116 corr,
6117 "dropping a route.bind response for a closing client; the module connection keeps serving"
6118 );
6119 return Ok(());
6120 }
6121 }
6122 Err(RouterError::Forwarding(err))
6123 }
6124
6125 fn handle_module_relay_response(
6126 &self,
6127 connection_id: ConnectionId,
6128 frame: Frame,
6129 ) -> Result<Vec<Frame>, RouterError> {
6130 let mut secondary_error = None;
6131 let outcome = match frame.header.ty {
6132 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6133 Ok(probe) if probe.op == "route.bind" => {
6134 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6135 Ok(ModuleControlResponse::RouteBindAck {}) => {
6136 RouteBindRelayOutcome::Accepted
6137 }
6138 Ok(other) => {
6139 let message =
6140 format!("route.bind response carried unexpected body: {other:?}");
6141 secondary_error = Some(control_error_frame(
6142 &frame,
6143 "invalid_control_body",
6144 message.clone(),
6145 )?);
6146 RouteBindRelayOutcome::ModuleGone(message)
6147 }
6148 Err(err) => {
6149 let message = format!("malformed route.bind response body: {err}");
6150 secondary_error = Some(control_error_frame(
6151 &frame,
6152 "invalid_control_body",
6153 message.clone(),
6154 )?);
6155 RouteBindRelayOutcome::ModuleGone(message)
6156 }
6157 }
6158 }
6159 Ok(probe) => {
6160 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6161 {
6162 Ok(response) => ModuleControlRpcOutcome::Response(response),
6163 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6164 "malformed {} response body: {err}",
6165 probe.op
6166 )),
6167 };
6168 let completion = self
6169 .forwarding
6170 .complete_module_control_rpc(
6171 connection_id,
6172 frame.header.corr,
6173 Some(&probe.op),
6174 outcome,
6175 )
6176 .map_err(RouterError::Forwarding)?;
6177 if !self.observe_module_control_completion(completion) {
6178 debug!(
6179 connection_id = connection_id.get(),
6180 corr = frame.header.corr,
6181 op = %probe.op,
6182 "dropping late or unknown module-control RPC response"
6183 );
6184 }
6185 return Ok(Vec::new());
6186 }
6187 Err(err) => {
6188 if let Some(expected_op) = self
6189 .forwarding
6190 .pending_module_control_op(connection_id, frame.header.corr)
6191 .map_err(RouterError::Forwarding)?
6192 {
6193 let completion = self
6194 .forwarding
6195 .complete_module_control_rpc(
6196 connection_id,
6197 frame.header.corr,
6198 None,
6199 ModuleControlRpcOutcome::MalformedResponse(format!(
6200 "malformed {expected_op} response body: {err}"
6201 )),
6202 )
6203 .map_err(RouterError::Forwarding)?;
6204 if !self.observe_module_control_completion(completion) {
6205 debug!(
6206 connection_id = connection_id.get(),
6207 corr = frame.header.corr,
6208 "dropping late malformed module-control RPC response"
6209 );
6210 }
6211 return Ok(Vec::new());
6212 }
6213 let message = format!("malformed route.bind response body: {err}");
6214 secondary_error = Some(control_error_frame(
6215 &frame,
6216 "invalid_control_body",
6217 message.clone(),
6218 )?);
6219 RouteBindRelayOutcome::ModuleGone(message)
6220 }
6221 },
6222 FrameType::Error => {
6223 if self
6224 .forwarding
6225 .pending_module_control_op(connection_id, frame.header.corr)
6226 .map_err(RouterError::Forwarding)?
6227 .is_some()
6228 {
6229 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6230 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6231 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6232 "malformed module-control ERROR body: {err}"
6233 )),
6234 };
6235 let completion = self
6236 .forwarding
6237 .complete_module_control_rpc(
6238 connection_id,
6239 frame.header.corr,
6240 None,
6241 outcome,
6242 )
6243 .map_err(RouterError::Forwarding)?;
6244 if !self.observe_module_control_completion(completion) {
6245 debug!(
6246 connection_id = connection_id.get(),
6247 corr = frame.header.corr,
6248 "dropping late or unknown module-control RPC error"
6249 );
6250 }
6251 return Ok(Vec::new());
6252 }
6253 match serde_json::from_slice::<ErrorBody>(&frame.body) {
6254 Ok(body) => RouteBindRelayOutcome::Rejected(body),
6255 Err(err) => {
6256 let message = format!("malformed route.bind ERROR body: {err}");
6257 secondary_error = Some(control_error_frame(
6258 &frame,
6259 "invalid_control_body",
6260 message.clone(),
6261 )?);
6262 RouteBindRelayOutcome::ModuleGone(message)
6263 }
6264 }
6265 }
6266 ty => {
6267 return Ok(vec![control_error_frame(
6268 &frame,
6269 "unsupported_control_frame",
6270 format!("unsupported module channel-0 frame {ty:?}"),
6271 )?])
6272 }
6273 };
6274
6275 let settled =
6276 self.forwarding
6277 .complete_pending_relay(connection_id, frame.header.corr, outcome);
6278 let completion = match settled {
6279 Ok(completion) => completion,
6280 Err(err) => {
6281 self.refuse_to_end_module_connection_for_a_client(
6282 connection_id,
6283 frame.header.corr,
6284 err,
6285 )?;
6286 return Ok(secondary_error.into_iter().collect());
6287 }
6288 };
6289 if let Some(target) = completion.abandoned.as_ref() {
6290 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6291 }
6292 if !completion.settled {
6293 debug!(
6294 connection_id = connection_id.get(),
6295 corr = frame.header.corr,
6296 frame_type = ?frame.header.ty,
6297 "dropping late or unknown route.bind relay response"
6298 );
6299 }
6300 Ok(secondary_error.into_iter().collect())
6301 }
6302
6303 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6304 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6305 self.cleanup_connection_with_end_reason(
6309 connection_id,
6310 RegistrationEndReason::ExplicitGoodbye,
6311 )
6312 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6313 Ok(Vec::new())
6314 }
6315}
6316
6317impl Default for ControlHandler {
6318 fn default() -> Self {
6319 Self::new(Arc::new(Registry::default()))
6320 }
6321}
6322
6323impl crate::supervise::SwapPromotionObserver for ControlHandler {
6324 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6325 self.apply_registration_capabilities(registration);
6326 }
6327}
6328
6329fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6330 CapabilityRequirementStatus {
6331 consumer: status.consumer,
6332 capability: status.capability,
6333 need: match status.need {
6334 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6335 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6336 },
6337 verdict: status.verdict.as_str().to_string(),
6338 episode_seq: status.episode_seq,
6339 config_satisfiable: status.config_satisfiable,
6340 runtime_available: status.runtime_available,
6341 detail: status.detail,
6342 }
6343}
6344
6345fn append_capability_problem_detail(
6346 detail: Option<String>,
6347 capability_detail: Option<String>,
6348) -> Option<String> {
6349 match (detail, capability_detail) {
6350 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6351 (Some(detail), None) => Some(detail),
6352 (None, Some(capability_detail)) => Some(capability_detail),
6353 (None, None) => None,
6354 }
6355}
6356
6357fn subc_ops() -> Vec<String> {
6358 SUBC_CONTROL_OPS
6359 .iter()
6360 .map(|op| (*op).to_string())
6361 .collect()
6362}
6363
6364fn module_subc_ops() -> Vec<String> {
6365 SUBC_CONTROL_OPS
6366 .iter()
6367 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6368 .map(|op| (*op).to_string())
6369 .collect()
6370}
6371
6372#[cfg(test)]
6373fn module_baseline_control_ops() -> Vec<String> {
6374 MODULE_BASELINE_CONTROL_OPS
6375 .iter()
6376 .map(|op| (*op).to_string())
6377 .collect()
6378}
6379
6380fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6381 let mut seen = HashSet::new();
6382 let mut effective = Vec::new();
6383 for op in MODULE_BASELINE_CONTROL_OPS {
6384 if seen.insert((*op).to_string()) {
6385 effective.push((*op).to_string());
6386 }
6387 }
6388 for op in declared.unwrap_or_default() {
6389 if seen.insert(op.clone()) {
6390 effective.push(op);
6391 }
6392 }
6393 effective
6394}
6395
6396fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6397 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6398}
6399
6400fn target_module_id(target: &RouteTarget) -> &str {
6401 match target {
6402 RouteTarget::ToolProvider { module_id }
6403 | RouteTarget::ManagementSurface { module_id }
6404 | RouteTarget::InternalService { module_id, .. } => module_id,
6405 }
6406}
6407
6408fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6409 roles.iter().any(|role| match (target, role) {
6410 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6411 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6412 (
6413 RouteTarget::InternalService { service_id, .. },
6414 ProviderRole::InternalService {
6415 service_id: provided,
6416 ..
6417 },
6418 ) => service_id == provided,
6419 _ => false,
6420 })
6421}
6422
6423fn is_routable_role(role: &ProviderRole) -> bool {
6424 matches!(
6425 role,
6426 ProviderRole::ToolProvider { .. }
6427 | ProviderRole::ManagementSurface { .. }
6428 | ProviderRole::InternalService { .. }
6429 )
6430}
6431
6432#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6433enum ControlRequestBodyError {
6434 UnknownOp,
6435 InvalidBody,
6436}
6437
6438#[derive(Debug, Deserialize)]
6439struct ControlOpProbe {
6440 op: String,
6441}
6442
6443const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6446
6447fn is_known_module_push_op(body: &[u8]) -> bool {
6448 serde_json::from_slice::<ControlOpProbe>(body)
6449 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6450 .unwrap_or(false)
6451}
6452
6453fn is_known_module_request_op(body: &[u8]) -> bool {
6454 serde_json::from_slice::<ControlOpProbe>(body)
6455 .map(|probe| is_module_to_subc_op(&probe.op))
6456 .unwrap_or(false)
6457}
6458
6459fn is_module_to_subc_op(op: &str) -> bool {
6460 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6461}
6462
6463fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6464 debug!(
6465 op = %op,
6466 connection_id = connection_id.get(),
6467 corr,
6468 "control dispatch"
6469 );
6470}
6471
6472fn log_slow_control_dispatch(
6473 dispatch_started_at: Option<StdInstant>,
6474 op: &'static str,
6475 connection_id: ConnectionId,
6476 corr: u64,
6477) {
6478 let Some(dispatch_started_at) = dispatch_started_at else {
6479 return;
6480 };
6481 let elapsed = dispatch_started_at.elapsed();
6482 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6483 warn!(
6484 op = %op,
6485 connection_id = connection_id.get(),
6486 corr,
6487 elapsed_ms = elapsed.as_millis() as u64,
6488 "slow control dispatch"
6489 );
6490 }
6491}
6492
6493fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6494 match request {
6495 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6496 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6497 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6498 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6499 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6500 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6501 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6502 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6503 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6504 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6505 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6506 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6507 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6508 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6509 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6510 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6511 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6512 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6513 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6514 }
6515}
6516
6517fn principal_label(principal: &Principal) -> String {
6518 match principal {
6519 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6520 Principal::Direct => "direct".to_string(),
6521 other => format!("{other:?}"),
6522 }
6523}
6524
6525fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6526 match request {
6527 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6528 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6529 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6530 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6531 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6532 }
6533}
6534
6535fn parse_client_control_request(
6536 body: &[u8],
6537) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6538 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6539 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6540 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6541 ControlRequestBodyError::InvalidBody
6542 }
6543 Ok(_) => ControlRequestBodyError::UnknownOp,
6544 Err(_) => ControlRequestBodyError::InvalidBody,
6545 };
6546 (err, classification)
6547 })
6548}
6549
6550fn parse_module_control_request_from_module(
6551 body: &[u8],
6552) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6553 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6554 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6555 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6556 Ok(_) => ControlRequestBodyError::UnknownOp,
6557 Err(_) => ControlRequestBodyError::InvalidBody,
6558 };
6559 (err, classification)
6560 })
6561}
6562
6563#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6564enum ProviderRoleKind {
6565 ToolProvider,
6566 PipelineStage,
6567 ManagementSurface,
6568 InternalService,
6569}
6570
6571fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6572 match role {
6573 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6574 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6575 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6576 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
6577 }
6578}
6579
6580fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
6581 roles.iter().map(provider_role_kind).collect()
6582}
6583
6584const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
6587
6588#[derive(Debug, Default, PartialEq, Eq)]
6590struct ScopeOutcomeCounts {
6591 created: usize,
6592 replaced: usize,
6593 updated: usize,
6594 unchanged: usize,
6595 refused: usize,
6596}
6597
6598impl ScopeOutcomeCounts {
6599 fn of(results: &[ScopeRecordResult]) -> Self {
6600 let mut counts = Self::default();
6601 for result in results {
6602 let slot = match result.outcome {
6603 ScopeRecordOutcome::Created => &mut counts.created,
6604 ScopeRecordOutcome::Replaced => &mut counts.replaced,
6605 ScopeRecordOutcome::Updated => &mut counts.updated,
6606 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
6607 ScopeRecordOutcome::Refused => &mut counts.refused,
6608 };
6609 *slot += 1;
6610 }
6611 counts
6612 }
6613}
6614
6615#[cfg(test)]
6616mod scope_outcome_count_tests {
6617 use super::*;
6618
6619 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
6620 ScopeRecordResult {
6621 scope_ref: "r".to_string(),
6622 scope_epoch: 1,
6623 outcome,
6624 code: None,
6625 message: None,
6626 version: None,
6627 parent_state: None,
6628 }
6629 }
6630
6631 #[test]
6634 fn every_outcome_is_counted_in_its_own_field() {
6635 let results = [
6636 result(ScopeRecordOutcome::Created),
6637 result(ScopeRecordOutcome::Created),
6638 result(ScopeRecordOutcome::Replaced),
6639 result(ScopeRecordOutcome::Updated),
6640 result(ScopeRecordOutcome::Unchanged),
6641 result(ScopeRecordOutcome::Refused),
6642 result(ScopeRecordOutcome::Refused),
6643 result(ScopeRecordOutcome::Refused),
6644 ];
6645 assert_eq!(
6646 ScopeOutcomeCounts::of(&results),
6647 ScopeOutcomeCounts {
6648 created: 2,
6649 replaced: 1,
6650 updated: 1,
6651 unchanged: 1,
6652 refused: 3,
6653 }
6654 );
6655 }
6656}
6657
6658fn capability_census_trigger(
6662 old: Option<&CapabilityDeclarations>,
6663 new: Option<&CapabilityDeclarations>,
6664) -> bool {
6665 let old_provides = old
6666 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
6667 .unwrap_or_default();
6668 let old_denies = old
6669 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
6670 .unwrap_or_default();
6671 let new = new.cloned().unwrap_or(CapabilityDeclarations {
6672 provides: Vec::new(),
6673 requires: Vec::new(),
6674 must_never_reach: Vec::new(),
6675 });
6676
6677 new.provides
6678 .iter()
6679 .any(|capability| !old_provides.contains(capability))
6680 || new
6681 .must_never_reach
6682 .iter()
6683 .any(|capability| !old_denies.contains(capability))
6684}
6685
6686fn denied_capability<'a>(
6689 opening_manifest: &'a ModuleManifest,
6690 target_manifest: &ModuleManifest,
6691) -> Option<&'a str> {
6692 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
6693 let target_capabilities = target_manifest.capabilities.as_ref()?;
6694 opening_capabilities
6695 .must_never_reach
6696 .iter()
6697 .find(|denied| {
6698 target_capabilities
6699 .provides
6700 .iter()
6701 .any(|provided| provided == *denied)
6702 })
6703 .map(String::as_str)
6704}
6705
6706fn catalog_update_frozen_field_message(
6707 registered: &ModuleManifest,
6708 provides: &[ProviderRole],
6709) -> Option<String> {
6710 let old_has_provides = !registered.provides.is_empty();
6711 let new_has_provides = !provides.is_empty();
6712 if old_has_provides != new_has_provides {
6713 return Some(format!(
6714 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
6715 registered.module_id
6716 ));
6717 }
6718
6719 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
6720 return Some(format!(
6721 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
6722 registered.module_id
6723 ));
6724 }
6725
6726 let registered_concurrency = manifest_concurrency(registered);
6727 let mut candidate = registered.clone();
6728 candidate.provides = provides.to_vec();
6729 let candidate_concurrency = manifest_concurrency(&candidate);
6730 if candidate_concurrency != registered_concurrency {
6731 return Some(format!(
6732 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
6733 registered.module_id, registered_concurrency, candidate_concurrency
6734 ));
6735 }
6736
6737 None
6740}
6741
6742fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
6743 manifest.provides.iter().any(is_routable_role)
6744}
6745
6746fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
6752 manifest
6753 .provides
6754 .iter()
6755 .find_map(|provider| match provider {
6756 ProviderRole::ToolProvider { concurrency, .. }
6757 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
6758 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
6759 })
6760 .unwrap_or(Concurrency::ModuleManaged)
6761}
6762
6763fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
6770 let has_management_surface = manifest
6771 .provides
6772 .iter()
6773 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
6774 if !has_management_surface {
6775 return false;
6776 }
6777 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
6778 return false;
6779 };
6780 let Some(provides) = raw
6781 .get("manifest")
6782 .and_then(|manifest| manifest.get("provides"))
6783 .and_then(serde_json::Value::as_array)
6784 else {
6785 return false;
6786 };
6787 provides.iter().any(|role| {
6792 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
6793 && role.get("concurrency").is_none()
6794 })
6795}
6796
6797fn negotiate_version(peer_version: u8) -> Result<u8, String> {
6798 if peer_version != PROTOCOL_VERSION {
6799 return Err(format!(
6800 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
6801 ));
6802 }
6803 Ok(PROTOCOL_VERSION)
6804}
6805
6806fn pong(frame: &Frame) -> Result<Frame, RouterError> {
6807 Frame::build_with_version(
6808 response_version(frame),
6809 FrameType::Pong,
6810 frame.header.flags,
6811 0,
6812 0,
6813 frame.header.corr,
6814 Vec::new(),
6815 )
6816 .map_err(RouterError::FrameBuild)
6817}
6818
6819pub(crate) fn control_error_frame(
6820 frame: &Frame,
6821 code: &'static str,
6822 message: impl Into<String>,
6823) -> Result<Frame, RouterError> {
6824 control_error_body_frame(
6825 frame,
6826 ErrorBody {
6827 code: code.to_string(),
6828 message: message.into(),
6829 detail: None,
6830 },
6831 )
6832}
6833
6834fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
6835 let body = serde_json::to_vec(&error).map_err(|err| {
6836 RouterError::backend(
6837 0,
6838 frame.header.corr,
6839 format!("failed to encode control ERROR: {err}"),
6840 )
6841 })?;
6842
6843 Frame::build_with_version(
6844 response_version(frame),
6845 FrameType::Error,
6846 control_flags(),
6847 0,
6848 0,
6849 frame.header.corr,
6850 body,
6851 )
6852 .map_err(RouterError::FrameBuild)
6853}
6854
6855fn control_response_body_frame<T: Serialize>(
6856 frame: &Frame,
6857 reply: &T,
6858 label: &'static str,
6859) -> Result<Frame, RouterError> {
6860 let body = serde_json::to_vec(reply).map_err(|err| {
6861 RouterError::backend(
6862 0,
6863 frame.header.corr,
6864 format!("failed to encode {label}: {err}"),
6865 )
6866 })?;
6867
6868 Frame::build_with_version(
6869 response_version(frame),
6870 FrameType::Response,
6871 control_flags(),
6872 0,
6873 0,
6874 frame.header.corr,
6875 body,
6876 )
6877 .map_err(RouterError::FrameBuild)
6878}
6879
6880fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6910 match err {
6911 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6912 ForwardingError::NoModuleConnection => "target_unavailable",
6913 ForwardingError::ModuleReloading { .. } => "module_reloading",
6914 ForwardingError::ClientRouteChannelExhausted { .. }
6915 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6916 ForwardingError::StaleModuleEndpoint
6917 | ForwardingError::UnknownReservation { .. }
6918 | ForwardingError::ConnectionClosing { .. }
6919 | ForwardingError::ClientEgressClosed { .. }
6920 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
6921 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
6924 ForwardingError::RelayCorrelationExhausted
6925 | ForwardingError::RouteOpenBuild(_)
6926 | ForwardingError::Poisoned => "forwarding_error",
6927 }
6928}
6929
6930fn response_version(frame: &Frame) -> u8 {
6931 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
6932 frame.header.ver
6933 } else {
6934 PROTOCOL_VERSION
6935 }
6936}
6937
6938fn control_flags() -> Flags {
6939 Flags::new(false, Priority::Passive, false)
6940}
6941
6942fn send_goodbye_target_best_effort(
6947 counters: &DaemonCounters,
6948 target: &GoodbyeTarget,
6949 context: &'static str,
6950) {
6951 let Ok(frame) = Frame::build_with_version(
6952 target.negotiated_ver,
6953 FrameType::Goodbye,
6954 control_flags(),
6955 target.channel,
6956 target.epoch,
6957 0,
6958 Vec::new(),
6959 ) else {
6960 return;
6961 };
6962 crate::forwarding::send_module_route_goodbye(
6963 counters,
6964 &target.sink,
6965 frame,
6966 target.module_id.as_deref(),
6967 context,
6968 );
6969}
6970
6971pub(crate) fn send_route_control_pushes(
6972 forwarding: &ForwardingTable,
6973 routes: Vec<EndpointRoute>,
6974 push: ClientControlPush,
6975) {
6976 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
6977 for route in routes {
6978 let target = route.goodbye_target;
6979 if let Some((existing, channels)) = targets
6980 .iter_mut()
6981 .find(|(existing, _)| existing.connection_id == target.connection_id)
6982 {
6983 debug_assert_eq!(
6984 existing.negotiated_ver, target.negotiated_ver,
6985 "one connection cannot negotiate multiple frame versions"
6986 );
6987 if !channels.contains(&target.channel) {
6988 channels.push(target.channel);
6989 }
6990 continue;
6991 }
6992 let channel = target.channel;
6993 targets.push((target, vec![channel]));
6994 }
6995 for (target, mut channels) in targets {
6996 channels.sort_unstable();
6997 let mut push = push.clone();
6998 match &mut push {
6999 ClientControlPush::RouteClosing {
7000 channels: covered, ..
7001 }
7002 | ClientControlPush::RouteClosed {
7003 channels: covered, ..
7004 } => *covered = channels,
7005 }
7006 let body = match serde_json::to_vec(&push) {
7007 Ok(body) => body,
7008 Err(err) => {
7009 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7010 continue;
7011 }
7012 };
7013 let frame = match Frame::build_with_version(
7014 target.negotiated_ver,
7015 FrameType::Push,
7016 control_flags(),
7017 0,
7018 0,
7019 0,
7020 body.clone(),
7021 ) {
7022 Ok(frame) => frame,
7023 Err(err) => {
7024 warn!(
7025 route_channel = target.channel,
7026 error = %err,
7027 "failed to build route lifecycle control PUSH frame"
7028 );
7029 continue;
7030 }
7031 };
7032 if let Err(err) = target.sink.try_send(frame) {
7033 if target.close_on_delivery_failure() {
7034 warn!(
7035 target_connection_id = target.connection_id.get(),
7036 route_channel = target.channel,
7037 error = %err,
7038 "route lifecycle control PUSH was not delivered to client; closing target connection"
7039 );
7040 let _ = forwarding.escalate_client_delivery_failure(
7041 target.connection_id,
7042 target.channel,
7043 target.epoch,
7044 CloseReason::new(
7045 "route_lifecycle_push_delivery_failed",
7046 format!(
7047 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7048 target.channel
7049 ),
7050 ),
7051 crate::forwarding::UndeliveredFrame {
7052 module_id: target.module_id.as_deref(),
7053 sink: &target.sink,
7054 },
7055 );
7056 }
7057 }
7058 }
7059}
7060
7061#[cfg(test)]
7062mod tests {
7063 mod activity_read {
7064 include!("activity_read_tests.rs");
7065 }
7066
7067 mod lifecycle_activity {
7068 include!("control_lifecycle_tests.rs");
7069 }
7070 #[cfg(unix)]
7071 #[tokio::test]
7072 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7073 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7074 let path = dir.join("subc.jsonc");
7075 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7076 "program":"/bin/sleep","args":["60"],"protocol":"none",
7077 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7078 }}}).to_string()).unwrap();
7079 let mut configured = crate::daemon_config::load(&path)
7080 .unwrap()
7081 .unwrap()
7082 .modules
7083 .pop()
7084 .unwrap();
7085 let registry = std::sync::Arc::new(crate::Registry::default());
7086 let handle = crate::SupervisorHandle::new();
7087 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7088 .with_handle(handle.clone());
7089 let module = supervisor
7090 .supervise_configured_with_health(
7091 configured.module_spec(),
7092 true,
7093 configured.health.clone(),
7094 None,
7095 configured.restart,
7096 )
7097 .unwrap();
7098 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7099 let before = module.status().unwrap().pid;
7100 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7101 configured.health.cadence = std::time::Duration::from_secs(3600);
7102 let health_only = handler
7103 .reconcile_supervised_modules(
7104 &supervisor,
7105 vec![configured.clone()],
7106 false,
7107 Principal::Direct,
7108 )
7109 .await
7110 .unwrap();
7111 assert!(
7112 health_only.changed_pending_reload.is_empty(),
7113 "health policy is already applied live"
7114 );
7115 assert_eq!(module.status().unwrap().pid, before);
7116 assert_eq!(
7117 module.configuration().unwrap().1.http,
7118 configured.health.http
7119 );
7120 configured.args = vec!["61".into()];
7121 let launch = handler
7122 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7123 .await
7124 .unwrap();
7125 assert_eq!(launch.changed_pending_reload, ["stock"]);
7126 assert_eq!(
7127 module.status().unwrap().pid,
7128 before,
7129 "a launch edit is stored until reload"
7130 );
7131 module.drain().await.unwrap();
7132 }
7133 use cortexkit_test_support::ScratchDir;
7134 use std::{
7135 collections::BTreeMap,
7136 fmt,
7137 path::PathBuf,
7138 sync::{Arc, Mutex},
7139 time::Duration,
7140 };
7141
7142 use serde_json::{json, Value};
7143 use subc_protocol::{
7144 manifest::{
7145 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7146 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7147 },
7148 session::HealthStatus,
7149 FrameType,
7150 };
7151
7152 use super::*;
7153 use crate::{
7154 forwarding::{DataRoute, DataRouteState},
7155 registry::ChannelState,
7156 router::FrameSink,
7157 stderr_tail::DEFAULT_MAX_LINE_BYTES,
7158 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7159 RouteCtx, Router,
7160 };
7161 use tokio::{
7162 sync::mpsc,
7163 time::{sleep, Instant},
7164 };
7165 use tracing::{
7166 field::{Field, Visit},
7167 Event, Subscriber,
7168 };
7169 use tracing_subscriber::{layer::Context, prelude::*, Layer};
7170
7171 fn fake_aft_stub_path() -> PathBuf {
7190 let mut path = std::env::current_exe().expect("current_exe available in tests");
7191 path.pop(); path.pop(); path.push(if cfg!(windows) {
7194 "fake-aft-stub.exe"
7195 } else {
7196 "fake-aft-stub"
7197 });
7198 assert!(
7199 path.exists(),
7200 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7201 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7202 path.display()
7203 );
7204 path
7205 }
7206
7207 fn client_retries(code: &str) -> bool {
7212 subc_protocol::error_codes::is_retryable_route_open(code)
7213 }
7214
7215 #[test]
7226 fn retryability_of_forwarding_codes_matches_the_failure() {
7227 let transient = [
7230 ForwardingError::NoModuleConnection,
7231 ForwardingError::ModuleReloading {
7232 module_id: "m".into(),
7233 },
7234 ForwardingError::StaleModuleEndpoint,
7235 ForwardingError::UnknownReservation {
7236 client_channel: 1,
7237 module_channel: 1,
7238 },
7239 ForwardingError::ConnectionClosing {
7240 connection_id: ConnectionId::new(1),
7241 },
7242 ForwardingError::ClientEgressClosed {
7243 connection_id: ConnectionId::new(1),
7244 },
7245 ForwardingError::ModuleEgressUnavailable {
7246 connection_id: ConnectionId::new(1),
7247 },
7248 ];
7249 for err in transient {
7250 let code = forwarding_error_code(&err);
7251 assert!(
7252 client_retries(code),
7253 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7254 );
7255 }
7256
7257 let permanent = [
7262 ForwardingError::ConnectionRoleConflict {
7263 connection_id: ConnectionId::new(1),
7264 },
7265 ForwardingError::ClientRouteChannelExhausted {
7266 connection_id: ConnectionId::new(1),
7267 },
7268 ForwardingError::ModuleRouteChannelExhausted {
7269 endpoint: ModuleEndpointId {
7270 connection_id: ConnectionId::new(1),
7271 generation: 1,
7272 },
7273 },
7274 ForwardingError::RelayCorrelationExhausted,
7275 ForwardingError::RouteOpenBuild("x".into()),
7276 ForwardingError::Poisoned,
7277 ];
7278 for err in permanent {
7279 let code = forwarding_error_code(&err);
7280 assert!(
7281 !client_retries(code),
7282 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7283 );
7284 }
7285 }
7286
7287 #[tokio::test]
7315 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7316 let handler = ControlHandler::default();
7317 let frame =
7318 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7319
7320 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7324 assert_eq!(
7325 stamped,
7326 Principal::Direct,
7327 "a caller that proved nothing must not be stamped as a supervised module"
7328 );
7329
7330 let forged = handler
7335 .route_open_principal(
7336 &frame,
7337 Some(ConsumerIdentity {
7338 module_id: "aft".to_string(),
7339 launch_nonce: "not-a-real-nonce".to_string(),
7340 }),
7341 )
7342 .unwrap();
7343 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7344 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7345 }
7346
7347 #[test]
7364 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7365 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7366 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7367 let ClientControlRequest::RouteOpen {
7368 consumer_identity, ..
7369 } = parsed
7370 else {
7371 panic!("route.open body must parse as RouteOpen");
7372 };
7373 assert_eq!(
7374 consumer_identity,
7375 Some(ConsumerIdentity {
7376 module_id: "aft".to_string(),
7377 launch_nonce: "n".to_string(),
7378 }),
7379 "the wire field name must reach the value route_open_principal reads"
7380 );
7381 }
7382
7383 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7384 ModuleManifest::builder(module_id, "0.1.0")
7385 .protocol_ver(protocol_ver)
7386 .provides(vec![ProviderRole::ToolProvider {
7387 tools: vec![Tool {
7388 name: "read".to_string(),
7389 description: None,
7390 execution_mode: ExecutionMode::Pure,
7391 schema: json!({"type": "object"}),
7392 }],
7393 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7394 concurrency: Concurrency::ModuleManaged,
7395 emits_push: true,
7396 sub_supervises: true,
7397 }])
7398 .build()
7399 }
7400
7401 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7402 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7403 }
7404
7405 fn hello_frame_with_control_ops(
7406 module_id: &str,
7407 protocol_ver: u8,
7408 corr: u64,
7409 control_ops: Option<Vec<String>>,
7410 ) -> Frame {
7411 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7412 }
7413
7414 fn hello_frame_with_nonce(
7415 module_id: &str,
7416 protocol_ver: u8,
7417 corr: u64,
7418 launch_nonce: Option<&str>,
7419 ) -> Frame {
7420 hello_frame_full(
7421 module_id,
7422 protocol_ver,
7423 corr,
7424 None,
7425 launch_nonce.map(ToOwned::to_owned),
7426 )
7427 }
7428
7429 fn hello_frame_full(
7430 module_id: &str,
7431 protocol_ver: u8,
7432 corr: u64,
7433 control_ops: Option<Vec<String>>,
7434 launch_nonce: Option<String>,
7435 ) -> Frame {
7436 let body = serde_json::to_vec(&ModuleHelloBody {
7437 manifest: manifest(module_id, protocol_ver),
7438 protocol_ver,
7439 control_ops,
7440 launch_nonce,
7441 })
7442 .unwrap();
7443 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7444 }
7445
7446 fn non_routable_hello_frame_with_control_ops(
7447 module_id: &str,
7448 corr: u64,
7449 control_ops: Option<Vec<String>>,
7450 ) -> Frame {
7451 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
7452 manifest.provides.clear();
7453 let body = serde_json::to_vec(&ModuleHelloBody {
7454 manifest,
7455 protocol_ver: PROTOCOL_VERSION,
7456 control_ops,
7457 launch_nonce: None,
7458 })
7459 .unwrap();
7460 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7461 }
7462
7463 fn capability_grammar_hello_frame(
7464 capabilities: Value,
7465 runtime_computed: Option<Value>,
7466 corr: u64,
7467 ) -> Frame {
7468 let mut body = serde_json::to_value(ModuleHelloBody {
7469 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
7470 protocol_ver: PROTOCOL_VERSION,
7471 control_ops: None,
7472 launch_nonce: None,
7473 })
7474 .expect("HELLO body serializes");
7475 body["manifest"]["capabilities"] = capabilities;
7476 if let Some(runtime_computed) = runtime_computed {
7477 body["runtime_computed"] = runtime_computed;
7478 }
7479 Frame::build(
7480 FrameType::Hello,
7481 control_flags(),
7482 0,
7483 0,
7484 corr,
7485 serde_json::to_vec(&body).expect("HELLO body reserializes"),
7486 )
7487 .expect("HELLO frame builds")
7488 }
7489
7490 fn channel_request(channel: u16, corr: u64) -> Frame {
7491 Frame::build(
7492 FrameType::Request,
7493 Flags::new(true, Priority::Interactive, false),
7494 channel,
7495 0,
7496 corr,
7497 b"opaque".to_vec(),
7498 )
7499 .unwrap()
7500 }
7501
7502 fn route_ctx(
7503 connection_id: ConnectionId,
7504 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7505 let (tx, rx) = mpsc::channel(8);
7506 (
7507 RouteCtx {
7508 connection_id,
7509 egress: FrameSink::new(tx),
7510 },
7511 rx,
7512 )
7513 }
7514
7515 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7516 serde_json::from_slice(&frame.body).unwrap()
7517 }
7518
7519 async fn hello_via_sink(
7524 handler: &ControlHandler,
7525 ctx: &RouteCtx,
7526 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7527 hello: Frame,
7528 ) -> Frame {
7529 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7530 assert!(
7531 replies.is_empty(),
7532 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7533 );
7534 let ack = rx
7535 .try_recv()
7536 .expect("HELLO_ACK is queued on the module sink")
7537 .frame;
7538 assert_eq!(ack.header.ty, FrameType::HelloAck);
7539 ack
7540 }
7541
7542 fn parse_error(frame: &Frame) -> Value {
7543 serde_json::from_slice(&frame.body).unwrap()
7544 }
7545
7546 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7547 serde_json::from_slice(&frame.body).unwrap()
7548 }
7549
7550 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7551 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7552 route_channel,
7553 route_epoch: 0,
7554 kind,
7555 })
7556 .unwrap();
7557 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7558 }
7559
7560 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7561 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7562 module_id: module_id.to_string(),
7563 })
7564 .unwrap();
7565 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7566 }
7567
7568 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7569 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7570 }
7571
7572 fn route_open_frame_with_consumer_capabilities(
7573 corr: u64,
7574 module_id: &str,
7575 project_root: ScratchDir,
7576 consumer_capabilities: Option<Vec<String>>,
7577 ) -> Frame {
7578 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7579 target: RouteTarget::ToolProvider {
7580 module_id: module_id.to_string(),
7581 },
7582 identity: BindIdentity::new(
7583 project_root.path().to_path_buf(),
7584 "unit".to_string(),
7585 "session".to_string(),
7586 ),
7587 consumer_identity: None,
7588 consumer_capabilities,
7589 role_versions: None,
7590 admission_facts: None,
7591 scope: None,
7592 })
7593 .unwrap();
7594 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7595 }
7596
7597 fn route_open_frame_with_role_versions(
7598 corr: u64,
7599 module_id: &str,
7600 project_root: ScratchDir,
7601 role_versions: Option<BTreeMap<String, String>>,
7602 ) -> Frame {
7603 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7604 target: RouteTarget::ToolProvider {
7605 module_id: module_id.to_string(),
7606 },
7607 identity: BindIdentity::new(
7608 project_root.path().to_path_buf(),
7609 "unit".to_string(),
7610 format!("session-{corr}"),
7611 ),
7612 consumer_identity: None,
7613 consumer_capabilities: None,
7614 role_versions,
7615 admission_facts: None,
7616 scope: None,
7617 })
7618 .unwrap();
7619 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7620 }
7621
7622 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
7623 entries
7624 .iter()
7625 .map(|(role, version)| (role.to_string(), version.to_string()))
7626 .collect()
7627 }
7628
7629 fn route_open_frame_with_admission_facts(
7630 corr: u64,
7631 module_id: &str,
7632 project_root: ScratchDir,
7633 consumer_identity: Option<subc_control::ConsumerIdentity>,
7634 facts: Option<Value>,
7635 ) -> Frame {
7636 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7637 target: RouteTarget::ToolProvider {
7638 module_id: module_id.to_string(),
7639 },
7640 identity: BindIdentity::new(
7641 project_root.path().to_path_buf(),
7642 "unit".to_string(),
7643 format!("session-{corr}"),
7644 ),
7645 consumer_identity,
7646 consumer_capabilities: None,
7647 role_versions: None,
7648 admission_facts: facts,
7649 scope: None,
7650 })
7651 .unwrap();
7652 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7653 }
7654
7655 #[derive(Clone, Default)]
7656 struct EventCapture {
7657 events: Arc<Mutex<Vec<CapturedEvent>>>,
7658 }
7659
7660 #[derive(Clone, Debug)]
7661 struct CapturedEvent {
7662 target: String,
7663 level: tracing::Level,
7664 fields: BTreeMap<String, String>,
7665 }
7666
7667 impl EventCapture {
7668 fn events(&self) -> Vec<CapturedEvent> {
7669 self.events.lock().unwrap().clone()
7670 }
7671 }
7672
7673 impl<S> Layer<S> for EventCapture
7674 where
7675 S: Subscriber,
7676 {
7677 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
7678 let mut visitor = EventFieldVisitor::default();
7679 event.record(&mut visitor);
7680 self.events.lock().unwrap().push(CapturedEvent {
7681 target: event.metadata().target().to_string(),
7682 level: *event.metadata().level(),
7683 fields: visitor.fields,
7684 });
7685 }
7686 }
7687
7688 #[derive(Default)]
7689 struct EventFieldVisitor {
7690 fields: BTreeMap<String, String>,
7691 }
7692
7693 impl Visit for EventFieldVisitor {
7694 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
7695 self.fields
7696 .insert(field.name().to_string(), format!("{value:?}"));
7697 }
7698 }
7699
7700 fn health_response(corr: u64, status: HealthStatus) -> Frame {
7701 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
7702 status,
7703 detail: Some("warming".to_string()),
7704 metrics: Some(json!({"queue_depth": 3})),
7705 })
7706 .unwrap();
7707 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7708 }
7709
7710 fn route_bind_ack(corr: u64) -> Frame {
7711 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
7712 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7713 }
7714
7715 fn unique_project_root(label: &str) -> ScratchDir {
7716 ScratchDir::new(label)
7717 }
7718
7719 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
7720 match parse_route_poll(frame) {
7721 ClientControlResponse::RoutePoll {
7722 status: None,
7723 live: Some(live),
7724 ..
7725 } => assert_eq!(live, expected_live),
7726 other => panic!("unexpected route.poll response: {other:?}"),
7727 }
7728 }
7729
7730 fn bind_liveness_route(
7731 registry: &Registry,
7732 forwarding: &ForwardingTable,
7733 module_id: &str,
7734 ) -> (RouteCtx, u16, u32) {
7735 let module_connection = ConnectionId::new(101);
7736 let client_connection = ConnectionId::new(202);
7737 let registration = registry
7738 .register_with_control_ops(
7739 manifest(module_id, PROTOCOL_VERSION),
7740 PROTOCOL_VERSION,
7741 module_connection,
7742 module_baseline_control_ops(),
7743 )
7744 .unwrap();
7745 let (module_tx, _module_rx) = mpsc::channel(8);
7746 let endpoint = forwarding
7747 .register_module_connection(
7748 module_connection,
7749 module_id.to_string(),
7750 PROTOCOL_VERSION,
7751 manifest_concurrency(®istration.manifest),
7752 FrameSink::new(module_tx),
7753 )
7754 .unwrap();
7755 let (client_ctx, _client_rx) = route_ctx(client_connection);
7756 let pending = forwarding
7757 .begin_route_bind_relay_for_test(
7758 client_connection,
7759 client_ctx.egress.clone(),
7760 1,
7761 module_id,
7762 )
7763 .unwrap();
7764 assert_eq!(pending.endpoint, endpoint);
7765 let route_channel = pending.client_channel;
7766 let route_epoch = pending.client_epoch;
7767 forwarding
7768 .complete_pending_relay(
7769 module_connection,
7770 pending.corr,
7771 RouteBindRelayOutcome::Accepted,
7772 )
7773 .unwrap();
7774 (client_ctx, route_channel, route_epoch)
7775 }
7776
7777 struct FakeProcessLiveness {
7778 live: Option<bool>,
7779 }
7780
7781 impl ModuleProcessLiveness for FakeProcessLiveness {
7782 fn process_live(&self, _module_id: &str) -> Option<bool> {
7783 self.live
7784 }
7785 }
7786
7787 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7788 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
7789 {
7790 let registry = Arc::new(Registry::default());
7791 let supervisor_handle = SupervisorHandle::new();
7792 let supervisor = Supervisor::new_for_test(
7793 Arc::clone(®istry),
7794 RestartPolicy::new(1, Duration::from_millis(10)),
7795 )
7796 .with_handle(supervisor_handle.clone());
7797 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
7798 let module = supervisor
7799 .spawn(ModuleSpec {
7800 module_id: "stderr-tail-wire".to_string(),
7801 program: fake_aft_stub_path(),
7802 args: Vec::new(),
7803 env: vec![
7804 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
7805 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
7806 ],
7807 reserved: false,
7808 reserved_prefixes: Vec::new(),
7809 protocol: ModuleProtocol::Subc,
7810 overlap: Default::default(),
7811 })
7812 .unwrap();
7813
7814 let deadline = Instant::now() + Duration::from_secs(5);
7815 loop {
7816 let tail = module.stderr_tail(None, None);
7817 if tail
7818 .entries
7819 .iter()
7820 .any(|entry| matches!(entry, TailEntry::ProcessStart))
7821 && tail.entries.iter().any(|entry| {
7822 matches!(
7823 entry,
7824 TailEntry::Line {
7825 truncated: true,
7826 ..
7827 }
7828 )
7829 })
7830 {
7831 break;
7832 }
7833 assert!(
7834 Instant::now() < deadline,
7835 "module did not produce a truncated line and restart boundary: {tail:?}"
7836 );
7837 sleep(Duration::from_millis(10)).await;
7838 }
7839
7840 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7841 let request = ClientControlRequest::SupervisorStderrTail {
7842 module_id: "stderr-tail-wire".to_string(),
7843 max_lines: None,
7844 max_bytes: None,
7845 };
7846 let frame = Frame::build(
7847 FrameType::Request,
7848 control_flags(),
7849 0,
7850 0,
7851 1,
7852 serde_json::to_vec(&request).unwrap(),
7853 )
7854 .unwrap();
7855 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7856 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7857 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
7858 serde_json::from_slice(&responses[0].body).unwrap()
7859 else {
7860 panic!("expected supervisor.stderr_tail response");
7861 };
7862
7863 assert!(
7864 tail.entries
7865 .iter()
7866 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
7867 "the control response lost the restart boundary"
7868 );
7869 let Some(StderrTailEntry::Line {
7870 text,
7871 truncated,
7872 at_ms,
7873 }) = tail.entries.iter().find(|entry| {
7874 matches!(
7875 entry,
7876 StderrTailEntry::Line {
7877 truncated: true,
7878 ..
7879 }
7880 )
7881 })
7882 else {
7883 panic!("the control response lost the truncated line");
7884 };
7885 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
7886 assert!(*truncated);
7887 assert!(
7888 at_ms.is_some(),
7889 "the control response lost the line's capture time"
7890 );
7891 }
7892
7893 #[tokio::test(flavor = "current_thread")]
7898 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
7899 let dir = ScratchDir::new("terminals-off-worker");
7900 let journal_path = dir.join("terminals.jsonl");
7901 let registry = Arc::new(Registry::default());
7902 let supervisor_handle = SupervisorHandle::new();
7903 let supervisor =
7904 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7905 .with_handle(supervisor_handle.clone())
7906 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7907 let module = supervisor
7908 .spawn(ModuleSpec {
7909 module_id: "terminal-off-worker".to_string(),
7910 program: fake_aft_stub_path(),
7911 args: Vec::new(),
7912 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7913 reserved: false,
7914 reserved_prefixes: Vec::new(),
7915 protocol: ModuleProtocol::Subc,
7916 overlap: Default::default(),
7917 })
7918 .unwrap();
7919 let deadline = Instant::now() + Duration::from_secs(5);
7920 while module.terminal_history().entries.len() != 2 {
7921 assert!(Instant::now() < deadline, "module did not record two exits");
7922 sleep(Duration::from_millis(10)).await;
7923 }
7924
7925 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
7926 let handler =
7927 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
7928 let frame = Frame::build(
7929 FrameType::Request,
7930 control_flags(),
7931 0,
7932 0,
7933 1,
7934 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
7935 module_id: "terminal-off-worker".to_string(),
7936 })
7937 .unwrap(),
7938 )
7939 .unwrap();
7940 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7941 let spawned_at = std::time::Instant::now();
7942 let read = tokio::spawn({
7943 let handler = Arc::clone(&handler);
7944 async move { handler.handle_control_frame(&ctx, frame).await }
7945 });
7946 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
7949 .await
7950 .unwrap()
7951 .expect("the history read reached its pause");
7952 let elapsed = spawned_at.elapsed();
7953 assert!(
7954 elapsed < Duration::from_secs(2) && !read.is_finished(),
7955 "this task could not run while the history read was paused \
7956 (resumed after {elapsed:?}, read finished: {})",
7957 read.is_finished()
7958 );
7959
7960 drop(release);
7961 let responses = read.await.unwrap().unwrap();
7962 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7963 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
7964 panic!("expected supervisor.terminals response");
7965 };
7966 assert_eq!(terminals.entries.len(), 2);
7967 assert_eq!(terminals.journal_skipped_lines, 0);
7968 assert_eq!(terminals.journal_read_errors, 0);
7969 }
7970
7971 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7972 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
7973 let registry = Arc::new(Registry::default());
7974 let supervisor_handle = SupervisorHandle::new();
7975 let supervisor =
7976 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7977 .with_handle(supervisor_handle.clone());
7978 let module = supervisor
7979 .spawn(ModuleSpec {
7980 module_id: "terminal-golden".to_string(),
7981 program: fake_aft_stub_path(),
7982 args: Vec::new(),
7983 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7984 reserved: false,
7985 reserved_prefixes: Vec::new(),
7986 protocol: ModuleProtocol::Subc,
7987 overlap: Default::default(),
7988 })
7989 .unwrap();
7990
7991 let deadline = Instant::now() + Duration::from_secs(5);
7992 while module.terminal_history().entries.len() != 2 {
7993 assert!(
7994 Instant::now() < deadline,
7995 "module did not retain two terminal exits: {:?}",
7996 module.terminal_history()
7997 );
7998 sleep(Duration::from_millis(10)).await;
7999 }
8000
8001 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8002 let request = ClientControlRequest::SupervisorTerminals {
8003 module_id: "terminal-golden".to_string(),
8004 };
8005 let frame = Frame::build(
8006 FrameType::Request,
8007 control_flags(),
8008 0,
8009 0,
8010 1,
8011 serde_json::to_vec(&request).unwrap(),
8012 )
8013 .unwrap();
8014 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8015 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8016 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8017 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8018 panic!("expected supervisor.terminals response");
8019 };
8020 assert_eq!(terminals.entries.len(), 2);
8021 assert_eq!(terminals.dropped, 0);
8022
8023 let mut rendered = serde_json::to_value(response).unwrap();
8024 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8027 for (index, entry) in rendered["entries"]
8028 .as_array_mut()
8029 .expect("terminal response entries array")
8030 .iter_mut()
8031 .enumerate()
8032 {
8033 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8034 }
8035
8036 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8037 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8038 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8039 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8040 std::fs::write(&golden_path, &serialized).unwrap();
8041 }
8042 let expected: Value =
8043 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8044 assert_eq!(rendered, expected);
8045 }
8046
8047 #[test]
8048 fn hello_registers_manifest_and_returns_ack() {
8049 let registry = Arc::new(Registry::default());
8050 let handler = ControlHandler::new(Arc::clone(®istry));
8051 let conn = ConnectionId::new(1);
8052
8053 let responses = handler
8054 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8055 .unwrap();
8056
8057 assert_eq!(responses.len(), 1);
8058 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8059 assert_eq!(responses[0].header.channel, 0);
8060 assert_eq!(responses[0].header.corr, 7);
8061 let ack = parse_ack(&responses[0]);
8062 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8063 assert!(ack
8064 .subc_capabilities
8065 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8066 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8067 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8068 assert!(ack
8069 .subc_ops
8070 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8071 assert!(ack
8072 .subc_ops
8073 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8074
8075 let registration = registry.get_module("aft").unwrap().unwrap();
8076 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8077 assert_eq!(registration.state, ChannelState::Active);
8078 assert_eq!(registration.connection_id, conn);
8079 assert_eq!(registration.control_ops, module_baseline_control_ops());
8080 }
8081
8082 #[test]
8083 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8084 let invalid_identifiers = [
8085 ("case_change", "credentials-Provider/v1"),
8086 ("leading_zero", "credentials-provider/v01"),
8087 ("trailing_hyphen", "credentials-provider-/v1"),
8088 ("consecutive_hyphens", "credentials--provider/v1"),
8089 ("uppercase", "Credentials-provider/v1"),
8090 ("missing_v", "credentials-provider/1"),
8091 ("whitespace", "credentials provider/v1"),
8092 ("zero_version", "credentials-provider/v0"),
8093 ("out_of_range_version", "credentials-provider/v4294967296"),
8094 (
8095 "overlength_name",
8096 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8097 ),
8098 ];
8099 let mut cases = invalid_identifiers
8100 .into_iter()
8101 .map(|(name, identifier)| {
8102 (
8103 format!("identifier_{name}"),
8104 "capabilities.provides[0]".to_string(),
8105 identifier.to_string(),
8106 json!({ "provides": [identifier] }),
8107 None,
8108 )
8109 })
8110 .collect::<Vec<_>>();
8111 cases.extend([
8112 (
8113 "unknown_need".to_string(),
8114 "capabilities.requires[0].need".to_string(),
8115 "deferred".to_string(),
8116 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8117 None,
8118 ),
8119 (
8120 "duplicate_provides".to_string(),
8121 "capabilities.provides[1]".to_string(),
8122 "credentials-provider/v1".to_string(),
8123 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8124 None,
8125 ),
8126 (
8127 "duplicate_must_never_reach".to_string(),
8128 "capabilities.must_never_reach[1]".to_string(),
8129 "credentials-provider/v1".to_string(),
8130 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8131 None,
8132 ),
8133 (
8134 "duplicate_requires_same_need".to_string(),
8135 "capabilities.requires[1]".to_string(),
8136 "credentials-provider/v1".to_string(),
8137 json!({ "requires": [
8138 { "capability": "credentials-provider/v1", "need": "required" },
8139 { "capability": "credentials-provider/v1", "need": "required" }
8140 ] }),
8141 None,
8142 ),
8143 (
8144 "duplicate_requires_conflicting_need".to_string(),
8145 "capabilities.requires[1]".to_string(),
8146 "credentials-provider/v1".to_string(),
8147 json!({ "requires": [
8148 { "capability": "credentials-provider/v1", "need": "required" },
8149 { "capability": "credentials-provider/v1", "need": "optional" }
8150 ] }),
8151 None,
8152 ),
8153 (
8154 "capabilities_root_pointer".to_string(),
8155 "runtime_computed[0]".to_string(),
8156 "/capabilities".to_string(),
8157 json!({}),
8158 Some(json!(["/capabilities"])),
8159 ),
8160 (
8161 "capabilities_descendant_pointer".to_string(),
8162 "runtime_computed[0]".to_string(),
8163 "/capabilities/provides".to_string(),
8164 json!({}),
8165 Some(json!(["/capabilities/provides"])),
8166 ),
8167 (
8168 "malformed_pointer_without_leading_slash".to_string(),
8169 "runtime_computed[0]".to_string(),
8170 "capabilities".to_string(),
8171 json!({}),
8172 Some(json!(["capabilities"])),
8173 ),
8174 (
8175 "malformed_pointer_escape".to_string(),
8176 "runtime_computed[0]".to_string(),
8177 "/roles/~2/tools".to_string(),
8178 json!({}),
8179 Some(json!(["/roles/~2/tools"])),
8180 ),
8181 (
8182 "unknown_capabilities_field".to_string(),
8183 "capabilities.future".to_string(),
8184 "<array>".to_string(),
8185 json!({ "future": [] }),
8186 None,
8187 ),
8188 ]);
8189
8190 for (index, (name, field, value, capabilities, runtime_computed)) in
8191 cases.into_iter().enumerate()
8192 {
8193 let registry = Arc::new(Registry::default());
8194 let handler = ControlHandler::new(Arc::clone(®istry));
8195 let response = handler
8196 .handle_control(
8197 ConnectionId::new((index + 1) as u64),
8198 capability_grammar_hello_frame(
8199 capabilities,
8200 runtime_computed,
8201 index as u64 + 1,
8202 ),
8203 )
8204 .expect("invalid HELLO returns a refusal");
8205
8206 assert_eq!(response.len(), 1, "{name} must emit one refusal");
8207 let error = parse_error(&response[0]);
8208 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8209 let message = error["message"]
8210 .as_str()
8211 .expect("error message is a string");
8212 assert!(
8213 message.contains(&field),
8214 "{name}: field missing from {message}"
8215 );
8216 assert!(
8217 message.contains(&value),
8218 "{name}: value missing from {message}"
8219 );
8220 assert_eq!(
8221 registry
8222 .active_registration_count()
8223 .expect("registry reads"),
8224 0,
8225 "{name}: refused HELLO must not create a catalog entry"
8226 );
8227 }
8228 }
8229
8230 #[test]
8231 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8232 let registry = Arc::new(Registry::default());
8233 let handler = ControlHandler::new(Arc::clone(®istry));
8234 let capabilities = json!({
8235 "provides": ["credentials-provider/v1"],
8236 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8237 "must_never_reach": ["federation-transport/v1"]
8238 });
8239 let response = handler
8240 .handle_control(
8241 ConnectionId::new(99),
8242 capability_grammar_hello_frame(
8243 capabilities.clone(),
8244 Some(json!(["/roles/0/tools"])),
8245 99,
8246 ),
8247 )
8248 .expect("valid HELLO registers");
8249 assert_eq!(response[0].header.ty, FrameType::HelloAck);
8250
8251 let request = Frame::build(
8252 FrameType::Request,
8253 control_flags(),
8254 0,
8255 0,
8256 100,
8257 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8258 .expect("catalog request serializes"),
8259 )
8260 .expect("catalog request frame builds");
8261 let response = handler
8262 .handle_catalog_list(request, None)
8263 .expect("catalog list succeeds");
8264 let ClientControlResponse::CatalogList { modules, .. } =
8265 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8266 else {
8267 panic!("catalog request must return catalog.list");
8268 };
8269 assert_eq!(modules.len(), 1);
8270 assert_eq!(
8271 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8272 capabilities
8273 );
8274 }
8275
8276 #[test]
8277 fn catalog_list_mirrors_management_operation_description() {
8278 let registry = Arc::new(Registry::default());
8279 let handler = ControlHandler::new(Arc::clone(®istry));
8280 let description = "List managed records and return their identifiers and metadata.";
8281 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8282 manifest.provides = vec![ProviderRole::ManagementSurface {
8283 operations: vec![ManagementOperation {
8284 name: "records.list".to_string(),
8285 kind: ManagementOperationKind::Query,
8286 description: Some(description.to_string()),
8287 }],
8288 config_schema: json!({"type": "object"}),
8289 observability: vec![ObservabilitySurface {
8290 name: "records.stats".to_string(),
8291 kind: ObservabilityKind::Snapshot,
8292 }],
8293 identity_scope: vec![IdentityScope::Project],
8294 concurrency: Concurrency::ModuleManaged,
8295 }];
8296 registry
8297 .register_with_control_ops(
8298 manifest,
8299 PROTOCOL_VERSION,
8300 ConnectionId::new(99),
8301 Vec::new(),
8302 )
8303 .expect("described management manifest registers");
8304
8305 let request = Frame::build(
8306 FrameType::Request,
8307 control_flags(),
8308 0,
8309 0,
8310 100,
8311 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8312 .expect("catalog request serializes"),
8313 )
8314 .expect("catalog request frame builds");
8315 let response = handler
8316 .handle_catalog_list(request, None)
8317 .expect("catalog list succeeds");
8318 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8319 assert_eq!(
8320 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8321 "catalog.list must preserve the declared operation description verbatim"
8322 );
8323 }
8324
8325 #[test]
8326 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8327 let registry = Arc::new(Registry::default());
8328 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8329 [("vault".to_string(), true), ("squatter".to_string(), true)],
8330 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8331 );
8332 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8333 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8334 provides: vec!["credentials-provider/v1".to_string()],
8335 requires: Vec::new(),
8336 must_never_reach: Vec::new(),
8337 });
8338 let frame = Frame::build(
8339 FrameType::Hello,
8340 control_flags(),
8341 0,
8342 0,
8343 77,
8344 serde_json::to_vec(&ModuleHelloBody {
8345 manifest: squatter,
8346 protocol_ver: PROTOCOL_VERSION,
8347 control_ops: None,
8348 launch_nonce: None,
8349 })
8350 .expect("HELLO serializes"),
8351 )
8352 .expect("HELLO frame builds");
8353 let response = handler
8354 .handle_control(ConnectionId::new(77), frame)
8355 .expect("reserved claim receives a typed refusal");
8356 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8357 assert_eq!(
8358 registry
8359 .active_registration_count()
8360 .expect("registry reads"),
8361 0,
8362 "a reserved capability refusal must not leave a catalog entry"
8363 );
8364 }
8365
8366 #[test]
8367 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8368 for settlement in ["timeout", "inconclusive", "drop"] {
8369 let breakers = RouteBindBreakers::default();
8370 let RouteBindAdmission::Admitted {
8371 guard: mut old,
8372 probe: false,
8373 } = breakers.admit("prov")
8374 else {
8375 panic!("ordinary relay admitted")
8376 };
8377 let RouteBindAdmission::Admitted {
8378 guard: mut opener, ..
8379 } = breakers.admit("prov")
8380 else {
8381 panic!("second relay admitted")
8382 };
8383 assert!(
8384 !opener
8385 .record_timeout(1, Duration::ZERO)
8386 .unwrap()
8387 .reopened_after_probe
8388 );
8389 let RouteBindAdmission::Admitted {
8390 guard: mut probe,
8391 probe: true,
8392 } = breakers.admit("prov")
8393 else {
8394 panic!("one cooldown probe admitted")
8395 };
8396 match settlement {
8397 "timeout" => assert!(
8398 !old.record_timeout(1, Duration::ZERO)
8399 .unwrap()
8400 .reopened_after_probe
8401 ),
8402 "inconclusive" => old.record_inconclusive(),
8403 "drop" => drop(old),
8404 _ => unreachable!(),
8405 }
8406 assert!(
8407 matches!(
8408 breakers.admit("prov"),
8409 RouteBindAdmission::Refused {
8410 probe_in_flight: true,
8411 ..
8412 }
8413 ),
8414 "{settlement} of a pre-open relay cannot release the real probe"
8415 );
8416 assert!(
8417 probe
8418 .record_timeout(1, Duration::ZERO)
8419 .unwrap()
8420 .reopened_after_probe
8421 );
8422 assert!(matches!(
8423 breakers.admit("prov"),
8424 RouteBindAdmission::Admitted { probe: true, .. }
8425 ));
8426 }
8427 let breakers = RouteBindBreakers::default();
8428 let admit = || match breakers.admit("prov") {
8429 RouteBindAdmission::Admitted { guard, .. } => guard,
8430 _ => panic!("relay admitted"),
8431 };
8432 admit().record_timeout(1, Duration::ZERO);
8433 let mut old_probe = admit();
8434 breakers.reset_for_new_module_connection("prov");
8435 admit().record_timeout(1, Duration::ZERO);
8436 let _new_probe = admit();
8437 old_probe.record_inconclusive();
8438 assert!(matches!(
8439 breakers.admit("prov"),
8440 RouteBindAdmission::Refused {
8441 probe_in_flight: true,
8442 ..
8443 }
8444 ));
8445 }
8446
8447 #[tokio::test]
8448 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
8449 for candidate in [false, true] {
8450 let registry = Arc::new(Registry::default());
8451 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8452 [("vault".to_string(), true), ("squatter".to_string(), true)],
8453 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8454 );
8455 let conn = ConnectionId::new(77);
8456 let (ctx, mut rx) = route_ctx(conn);
8457 let initial = capability_manifest("squatter", &[], &[]);
8458 if candidate {
8459 registry
8460 .register_candidate_with_control_ops(
8461 initial.clone(),
8462 PROTOCOL_VERSION,
8463 conn,
8464 module_baseline_control_ops(),
8465 )
8466 .unwrap();
8467 handler
8468 .forwarding
8469 .register_candidate_module_connection(
8470 conn,
8471 "squatter".to_string(),
8472 PROTOCOL_VERSION,
8473 manifest_concurrency(&initial),
8474 ctx.egress.clone(),
8475 )
8476 .unwrap();
8477 } else {
8478 hello_via_sink(
8479 &handler,
8480 &ctx,
8481 &mut rx,
8482 hello_frame_with_manifest(initial.clone(), 1),
8483 )
8484 .await;
8485 }
8486 let response = handler
8487 .handle_control_frame(
8488 &ctx,
8489 catalog_update_with_capabilities_frame(
8490 2,
8491 capability_manifest("squatter", &["credentials-provider/v1"], &[])
8492 .capabilities
8493 .unwrap(),
8494 ),
8495 )
8496 .await
8497 .unwrap();
8498 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8499 assert_eq!(
8500 registry
8501 .get_module_by_connection(conn)
8502 .unwrap()
8503 .unwrap()
8504 .manifest,
8505 initial
8506 );
8507 }
8508 }
8509
8510 #[test]
8511 fn server_describe_surfaces_required_capability_verdict_fields() {
8512 let registry = Arc::new(Registry::default());
8513 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8514 [
8515 ("consumer".to_string(), true),
8516 ("provider".to_string(), false),
8517 ],
8518 BTreeMap::new(),
8519 );
8520 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
8521 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8522 provides: Vec::new(),
8523 requires: vec![subc_protocol::manifest::CapabilityRequirement {
8524 capability: "credentials-provider/v1".to_string(),
8525 need: subc_protocol::manifest::CapabilityNeed::Required,
8526 }],
8527 must_never_reach: Vec::new(),
8528 });
8529 let hello = Frame::build(
8530 FrameType::Hello,
8531 control_flags(),
8532 0,
8533 0,
8534 78,
8535 serde_json::to_vec(&ModuleHelloBody {
8536 manifest: consumer,
8537 protocol_ver: PROTOCOL_VERSION,
8538 control_ops: None,
8539 launch_nonce: None,
8540 })
8541 .expect("HELLO serializes"),
8542 )
8543 .expect("HELLO frame builds");
8544 handler
8545 .handle_control(ConnectionId::new(78), hello)
8546 .expect("consumer registers");
8547 let describe = Frame::build(
8548 FrameType::Request,
8549 control_flags(),
8550 0,
8551 0,
8552 79,
8553 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8554 .expect("request serializes"),
8555 )
8556 .expect("describe frame builds");
8557 let response = handler
8558 .handle_server_describe(describe)
8559 .expect("server.describe succeeds");
8560 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8561 let requirement = &rendered["capability_requirements"][0];
8562 assert_eq!(requirement["consumer"], "consumer");
8563 assert_eq!(requirement["verdict"], "never_provided");
8564 assert_eq!(requirement["episode_seq"], 1);
8565 assert_eq!(requirement["config_satisfiable"], false);
8566 assert_eq!(requirement["runtime_available"], false);
8567 assert!(requirement["detail"]
8568 .as_str()
8569 .expect("detail string")
8570 .contains("credentials-provider/v1"));
8571 }
8572
8573 #[test]
8574 fn catalog_list_omits_capabilities_for_legacy_manifest() {
8575 let registry = Arc::new(Registry::default());
8576 let handler = ControlHandler::new(Arc::clone(®istry));
8577 let hello = handler
8578 .handle_control(
8579 ConnectionId::new(101),
8580 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
8581 )
8582 .expect("legacy HELLO registers");
8583 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
8584
8585 let request = Frame::build(
8586 FrameType::Request,
8587 control_flags(),
8588 0,
8589 0,
8590 102,
8591 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8592 .expect("catalog request serializes"),
8593 )
8594 .expect("catalog request frame builds");
8595 let response = handler
8596 .handle_catalog_list(request, None)
8597 .expect("catalog list succeeds");
8598 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8599 assert!(
8600 body["modules"][0].get("capabilities").is_none(),
8601 "legacy manifest must retain an absent capabilities field on catalog.list"
8602 );
8603 }
8604
8605 #[test]
8606 fn hello_ack_omits_storage_when_no_storage_config() {
8607 let registry = Arc::new(Registry::default());
8608 let handler = ControlHandler::new(Arc::clone(®istry));
8609 let responses = handler
8610 .handle_control(
8611 ConnectionId::new(1),
8612 hello_frame("aft", PROTOCOL_VERSION, 7),
8613 )
8614 .unwrap();
8615 let ack = parse_ack(&responses[0]);
8616 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
8617 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
8618 }
8619
8620 #[tokio::test]
8621 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
8622 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
8623 let registry = Arc::new(Registry::default());
8624 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
8625 let responses = handler
8626 .handle_control(
8627 ConnectionId::new(1),
8628 hello_frame("aft", PROTOCOL_VERSION, 7),
8629 )
8630 .unwrap();
8631 let ack = parse_ack(&responses[0]);
8632 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
8633
8634 let described = handler
8635 .handle_control_frame(
8636 &route_ctx(ConnectionId::new(2)).0,
8637 Frame::build(
8638 FrameType::Request,
8639 control_flags(),
8640 0,
8641 0,
8642 9,
8643 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8644 )
8645 .unwrap(),
8646 )
8647 .await
8648 .unwrap();
8649 let ClientControlResponse::ServerDescribe { machine_id, .. } =
8650 serde_json::from_slice(&described[0].body).unwrap()
8651 else {
8652 panic!("server.describe answered with another shape");
8653 };
8654 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
8655 }
8656
8657 #[test]
8658 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
8659 let registry = Arc::new(Registry::default());
8662 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
8663 crate::daemon_config::StorageConfig::Sqlite {
8664 data_home: std::path::PathBuf::from("/data"),
8665 },
8666 ));
8667
8668 let responses = handler
8669 .handle_control(
8670 ConnectionId::new(1),
8671 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
8672 )
8673 .unwrap();
8674 let ack = parse_ack(&responses[0]);
8675 assert_eq!(
8676 ack.storage,
8677 Some(serde_json::json!({
8678 "module_id": "alfonso-routing",
8679 "storage_namespace": "default",
8680 "isolation": { "kind": "module" },
8681 "backend": {
8682 "backend": "sqlite",
8683 "path": "/data/cortexkit/alfonso-routing/store.db"
8684 }
8685 })),
8686 "the delivered descriptor is the module's own sqlite store path"
8687 );
8688 }
8689
8690 #[test]
8691 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
8692 let registry = Arc::new(Registry::default());
8693 let handler = ControlHandler::new(Arc::clone(®istry));
8694 let conn = ConnectionId::new(1);
8695 let responses = handler
8696 .handle_control(
8697 conn,
8698 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8699 )
8700 .unwrap();
8701 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8702 let registration = registry.get_module("aft").unwrap().unwrap();
8703 assert_eq!(registration.control_ops, module_baseline_control_ops());
8704
8705 let frame =
8706 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
8707 assert!(handler
8708 .guard_module_control_op(&frame, "aft", "route.bind")
8709 .unwrap()
8710 .is_none());
8711 let error = handler
8712 .guard_module_control_op(&frame, "aft", "test.synthetic")
8713 .unwrap()
8714 .expect("synthetic ungranted op should be rejected");
8715 assert_eq!(error.header.ty, FrameType::Error);
8716 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
8717 }
8718
8719 #[test]
8720 fn hello_control_ops_some_adds_optional_grants() {
8721 let registry = Arc::new(Registry::default());
8722 let handler = ControlHandler::new(Arc::clone(®istry));
8723 handler
8724 .handle_control(
8725 ConnectionId::new(1),
8726 hello_frame_with_control_ops(
8727 "aft",
8728 PROTOCOL_VERSION,
8729 7,
8730 Some(vec![
8731 "future.synthetic".to_string(),
8732 "route.bind".to_string(),
8733 ]),
8734 ),
8735 )
8736 .unwrap();
8737 let registration = registry.get_module("aft").unwrap().unwrap();
8738 assert_eq!(
8739 registration.control_ops,
8740 vec![
8741 "route.bind".to_string(),
8742 "route.status".to_string(),
8743 "future.synthetic".to_string(),
8744 ]
8745 );
8746 let frame =
8747 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
8748 assert!(handler
8749 .guard_module_control_op(&frame, "aft", "future.synthetic")
8750 .unwrap()
8751 .is_none());
8752 }
8753
8754 #[tokio::test]
8755 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
8756 let registry = Arc::new(Registry::default());
8757 let forwarding = Arc::new(ForwardingTable::default());
8758 let handler =
8759 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8760 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
8761 hello_via_sink(
8762 &handler,
8763 &module_ctx,
8764 &mut module_rx,
8765 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8766 )
8767 .await;
8768
8769 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
8770 let responses = handler
8771 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
8772 .await
8773 .unwrap();
8774 assert_eq!(responses.len(), 1);
8775 assert_eq!(responses[0].header.ty, FrameType::Error);
8776 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
8777 assert!(module_rx.try_recv().is_err());
8778 }
8779
8780 #[tokio::test]
8781 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
8782 let registry = Arc::new(Registry::default());
8783 let forwarding = Arc::new(ForwardingTable::default());
8784 let handler =
8785 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8786 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
8787 hello_via_sink(
8788 &handler,
8789 &module_ctx,
8790 &mut module_rx,
8791 hello_frame_with_control_ops(
8792 "aft",
8793 PROTOCOL_VERSION,
8794 7,
8795 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8796 ),
8797 )
8798 .await;
8799
8800 let project_root = unique_project_root("demux");
8801 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
8802 let route_handler = handler.clone();
8803 let route_task = tokio::spawn(async move {
8804 route_handler
8805 .handle_control_frame(
8806 &route_client_ctx,
8807 route_open_frame(100, "aft", project_root),
8808 )
8809 .await
8810 .unwrap()
8811 });
8812 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8813 .await
8814 .unwrap()
8815 .unwrap();
8816 assert!(matches!(
8817 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
8818 ModuleControlRequest::RouteBind { .. }
8819 ));
8820
8821 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
8822 let health_handler = handler.clone();
8823 let health_task = tokio::spawn(async move {
8824 health_handler
8825 .handle_control_frame(
8826 &health_client_ctx,
8827 supervisor_health_probe_frame(101, "aft"),
8828 )
8829 .await
8830 .unwrap()
8831 });
8832 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8833 .await
8834 .unwrap()
8835 .unwrap();
8836 assert_eq!(
8837 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8838 ModuleControlRequest::HealthCheck {}
8839 );
8840
8841 handler
8842 .handle_control_frame(
8843 &module_ctx,
8844 health_response(health_frame.header.corr, HealthStatus::Degraded),
8845 )
8846 .await
8847 .unwrap();
8848 let health_response = health_task.await.unwrap();
8849 assert_eq!(health_response.len(), 1);
8850 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8851 ClientControlResponse::SupervisorHealthProbe {
8852 module_id,
8853 status,
8854 detail,
8855 metrics,
8856 } => {
8857 assert_eq!(module_id, "aft");
8858 assert_eq!(status, HealthStatus::Degraded);
8859 assert_eq!(detail.as_deref(), Some("warming"));
8860 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
8861 }
8862 other => panic!("unexpected health response: {other:?}"),
8863 }
8864
8865 handler
8866 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8867 .await
8868 .unwrap();
8869 let route_response = route_task.await.unwrap();
8870 assert!(route_response.is_empty());
8871 let published = route_client_rx.recv().await.unwrap();
8872 assert!(matches!(
8873 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8874 ClientControlResponse::RouteOpen { .. }
8875 ));
8876 }
8877
8878 async fn relay_route_open(
8883 handler: &ControlHandler,
8884 client_connection: ConnectionId,
8885 client_egress: &FrameSink,
8886 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8887 corr: u64,
8888 module_id: &str,
8889 project_root_label: &str,
8890 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
8891 let ctx = RouteCtx {
8892 connection_id: client_connection,
8893 egress: client_egress.clone(),
8894 };
8895 let handler = handler.clone();
8896 let project_root = unique_project_root(project_root_label);
8897 let module_id = module_id.to_string();
8898 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
8899 let task = tokio::spawn(async move {
8900 let _guard = tracing::dispatcher::set_default(&dispatch);
8901 handler
8902 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8903 .await
8904 .unwrap()
8905 });
8906 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8907 .await
8908 .expect("module receives the relayed route.bind")
8909 .expect("module egress is open");
8910 (task, bind.frame)
8911 }
8912
8913 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8914 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8915 ModuleControlRequest::RouteBind {
8916 route_channel,
8917 epoch,
8918 ..
8919 } => (route_channel, epoch),
8920 other => panic!("expected a route.bind request, got {other:?}"),
8921 }
8922 }
8923
8924 fn published_route(frame: &Frame) -> (u16, u32) {
8925 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
8926 ClientControlResponse::RouteOpen {
8927 route_channel,
8928 route_epoch,
8929 } => (route_channel, route_epoch),
8930 other => panic!("expected a route.open response, got {other:?}"),
8931 }
8932 }
8933
8934 #[tokio::test]
8954 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
8955 let registry = Arc::new(Registry::default());
8956 let forwarding = Arc::new(ForwardingTable::default());
8957 let handler =
8958 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8959
8960 let module_connection = ConnectionId::new(30);
8961 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8962 hello_via_sink(
8963 &handler,
8964 &module_ctx,
8965 &mut module_rx,
8966 hello_frame("aft", PROTOCOL_VERSION, 7),
8967 )
8968 .await;
8969
8970 let dying_client = ConnectionId::new(31);
8971 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
8972
8973 let (first_task, first_bind) = relay_route_open(
8976 &handler,
8977 dying_client,
8978 &dying_ctx.egress,
8979 &mut module_rx,
8980 100,
8981 "aft",
8982 "closing-first",
8983 )
8984 .await;
8985 handler
8986 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
8987 .await
8988 .unwrap();
8989 assert!(first_task.await.unwrap().is_empty());
8990 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
8991
8992 let (second_task, second_bind) = relay_route_open(
8994 &handler,
8995 dying_client,
8996 &dying_ctx.egress,
8997 &mut module_rx,
8998 101,
8999 "aft",
9000 "closing-second",
9001 )
9002 .await;
9003 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9004
9005 assert!(forwarding
9008 .escalate_client_delivery_failure(
9009 dying_client,
9010 first_channel,
9011 first_epoch,
9012 CloseReason::new(
9013 "module_to_client_delivery_failed",
9014 "client egress refused a module frame",
9015 ),
9016 crate::forwarding::UndeliveredFrame {
9017 module_id: None,
9018 sink: &dying_ctx.egress,
9019 },
9020 )
9021 .unwrap());
9022 assert!(!dying_ctx.egress.is_closed());
9023
9024 let ack = handler
9026 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9027 .await;
9028 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9029 if module_loop_error.is_some() {
9030 handler.cleanup_connection(module_connection).unwrap();
9034 }
9035 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9039 .await
9040 .ok()
9041 .flatten();
9042
9043 assert!(
9045 registry
9046 .get_module_by_connection(module_connection)
9047 .unwrap()
9048 .is_some(),
9049 "one client's closing connection ended the shared module connection: \
9050 {module_loop_error:?}"
9051 );
9052 let cotenant = ConnectionId::new(32);
9054 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9055 let (cotenant_task, cotenant_bind) = relay_route_open(
9056 &handler,
9057 cotenant,
9058 &cotenant_ctx.egress,
9059 &mut module_rx,
9060 102,
9061 "aft",
9062 "closing-cotenant",
9063 )
9064 .await;
9065 handler
9066 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9067 .await
9068 .unwrap();
9069 assert!(cotenant_task.await.unwrap().is_empty());
9070 let (cotenant_channel, cotenant_epoch) =
9071 published_route(&cotenant_rx.recv().await.unwrap());
9072 assert!(matches!(
9073 forwarding
9074 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9075 .unwrap(),
9076 DataRoute::Client(DataRouteState::Bound(_))
9077 ));
9078
9079 let goodbye = post_ack_module_frame
9082 .expect("module receives a GOODBYE for the abandoned route channel");
9083 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9084 assert_eq!(goodbye.header.channel, abandoned_channel);
9085 assert_eq!(goodbye.header.epoch, abandoned_epoch);
9086
9087 assert!(dying_rx.try_recv().is_err());
9091 let second_response = second_task.await.unwrap();
9092 assert_eq!(second_response.len(), 1);
9093 assert_eq!(
9094 parse_error(&second_response[0])["code"],
9095 "target_unavailable"
9096 );
9097 }
9098
9099 async fn route_open_held_after_bind_send() -> (
9103 ControlHandler,
9104 Arc<tokio::sync::Barrier>,
9105 RouteCtx,
9106 mpsc::Receiver<crate::router::OutboundFrame>,
9107 ) {
9108 let hold = Arc::new(tokio::sync::Barrier::new(2));
9109 let handler = ControlHandler::with_forwarding(
9110 Arc::new(Registry::default()),
9111 Arc::new(ForwardingTable::default()),
9112 )
9113 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
9114 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9115 hello_via_sink(
9116 &handler,
9117 &module_ctx,
9118 &mut module_rx,
9119 hello_frame("aft", PROTOCOL_VERSION, 7),
9120 )
9121 .await;
9122 (handler, hold, module_ctx, module_rx)
9123 }
9124
9125 #[tokio::test]
9132 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
9133 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9134 let client = ConnectionId::new(41);
9135 let (client_ctx, mut client_rx) = route_ctx(client);
9136
9137 let (open_task, bind) = relay_route_open(
9138 &handler,
9139 client,
9140 &client_ctx.egress,
9141 &mut module_rx,
9142 200,
9143 "aft",
9144 "acked-before-mark",
9145 )
9146 .await;
9147 let (module_channel, module_epoch) = route_bind_channel(&bind);
9148 hold.wait().await;
9150
9151 assert!(handler
9153 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9154 .await
9155 .unwrap()
9156 .is_empty());
9157 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
9158
9159 hold.wait().await;
9160 assert!(
9161 open_task.await.unwrap().is_empty(),
9162 "route.open answers accepted"
9163 );
9164
9165 match module_rx.try_recv() {
9168 Err(mpsc::error::TryRecvError::Empty) => {}
9169 Ok(outbound) => panic!(
9170 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
9171 outbound.frame.header.ty,
9172 outbound.frame.header.channel,
9173 outbound.frame.header.epoch
9174 ),
9175 Err(err) => panic!("module egress closed: {err}"),
9176 }
9177
9178 let router = Router::with_control_handler(Arc::new(handler.clone()));
9181 let request = Frame::build(
9182 FrameType::Request,
9183 Flags::new(true, Priority::Interactive, false),
9184 client_channel,
9185 client_epoch,
9186 201,
9187 b"first request".to_vec(),
9188 )
9189 .unwrap();
9190 router
9191 .route_for_connection(&client_ctx, request)
9192 .await
9193 .unwrap();
9194 let delivered = module_rx.try_recv().unwrap().frame;
9195 assert_eq!(delivered.header.ty, FrameType::Request);
9196 assert_eq!(
9197 (delivered.header.channel, delivered.header.epoch),
9198 (module_channel, module_epoch)
9199 );
9200 assert_eq!(delivered.body, b"first request");
9201 }
9202
9203 #[tokio::test]
9209 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
9210 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9211 let client = ConnectionId::new(42);
9212 let (client_ctx, _client_rx) = route_ctx(client);
9213
9214 let (open_task, bind) = relay_route_open(
9215 &handler,
9216 client,
9217 &client_ctx.egress,
9218 &mut module_rx,
9219 300,
9220 "aft",
9221 "released-before-mark",
9222 )
9223 .await;
9224 let (module_channel, module_epoch) = route_bind_channel(&bind);
9225 hold.wait().await;
9226
9227 handler.cleanup_connection(client).unwrap();
9228 assert!(
9229 module_rx.try_recv().is_err(),
9230 "client teardown must not GOODBYE a bind route.open is still queueing"
9231 );
9232
9233 hold.wait().await;
9234 let response = open_task.await.unwrap();
9235 assert_eq!(response.len(), 1);
9236 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9237
9238 let goodbye = module_rx
9239 .try_recv()
9240 .expect("module is told to drop the binding the daemon abandoned")
9241 .frame;
9242 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9243 assert_eq!(
9244 (goodbye.header.channel, goodbye.header.epoch),
9245 (module_channel, module_epoch)
9246 );
9247 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9248 }
9249
9250 #[tokio::test]
9255 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
9256 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9257 let client = ConnectionId::new(43);
9258 let (client_ctx, client_rx) = route_ctx(client);
9259
9260 let (open_task, bind) = relay_route_open(
9261 &handler,
9262 client,
9263 &client_ctx.egress,
9264 &mut module_rx,
9265 400,
9266 "aft",
9267 "gone-client-before-mark",
9268 )
9269 .await;
9270 let (module_channel, module_epoch) = route_bind_channel(&bind);
9271 hold.wait().await;
9272
9273 drop(client_rx);
9274 assert!(client_ctx.egress.is_closed());
9275 handler
9276 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9277 .await
9278 .unwrap();
9279
9280 hold.wait().await;
9281 let response = open_task.await.unwrap();
9282 assert_eq!(response.len(), 1);
9283 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9284
9285 let goodbye = module_rx
9286 .try_recv()
9287 .expect("module is told to drop the binding it acked")
9288 .frame;
9289 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9290 assert_eq!(
9291 (goodbye.header.channel, goodbye.header.epoch),
9292 (module_channel, module_epoch)
9293 );
9294 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9295 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
9296 }
9297
9298 #[test]
9305 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
9306 let handler = ControlHandler::default();
9307 let module_connection = ConnectionId::new(30);
9308 let client_connection = ConnectionId::new(31);
9309
9310 handler
9311 .refuse_to_end_module_connection_for_a_client(
9312 module_connection,
9313 77,
9314 ForwardingError::ConnectionClosing {
9315 connection_id: client_connection,
9316 },
9317 )
9318 .expect("a closing client must never end the module connection");
9319
9320 assert!(matches!(
9321 handler.refuse_to_end_module_connection_for_a_client(
9322 module_connection,
9323 78,
9324 ForwardingError::ConnectionClosing {
9325 connection_id: module_connection,
9326 },
9327 ),
9328 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
9329 connection_id
9330 })) if connection_id == module_connection
9331 ));
9332 assert!(matches!(
9333 handler.refuse_to_end_module_connection_for_a_client(
9334 module_connection,
9335 79,
9336 ForwardingError::Poisoned,
9337 ),
9338 Err(RouterError::Forwarding(ForwardingError::Poisoned))
9339 ));
9340 assert!(matches!(
9341 handler.refuse_to_end_module_connection_for_a_client(
9342 module_connection,
9343 80,
9344 ForwardingError::StaleModuleEndpoint,
9345 ),
9346 Err(RouterError::Forwarding(
9347 ForwardingError::StaleModuleEndpoint
9348 ))
9349 ));
9350 }
9351
9352 #[tokio::test]
9365 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
9366 let registry = Arc::new(Registry::default());
9367 let forwarding = Arc::new(ForwardingTable::default());
9368 let supervisor = SupervisorHandle::new();
9369 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9370 let handler =
9371 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9372 .with_supervisor(supervisor);
9373
9374 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9375 hello_via_sink(
9376 &handler,
9377 &target_ctx,
9378 &mut target_rx,
9379 hello_frame("target", PROTOCOL_VERSION, 1),
9380 )
9381 .await;
9382
9383 let wrong_nonce = handler
9387 .handle_control_frame(
9388 &route_ctx(ConnectionId::new(91)).0,
9389 route_open_frame_with_admission_facts(
9390 20,
9391 "target",
9392 unique_project_root("admission-facts"),
9393 Some(subc_control::ConsumerIdentity {
9394 module_id: "fed".to_string(),
9395 launch_nonce: "not-the-real-nonce".to_string(),
9396 }),
9397 None,
9398 ),
9399 )
9400 .await
9401 .unwrap();
9402 assert_eq!(
9403 parse_error(&wrong_nonce[0])["code"],
9404 "bad_consumer_identity",
9405 "a mismatched launch nonce must be refused, not stamped Reserved"
9406 );
9407
9408 let never_spawned = handler
9412 .handle_control_frame(
9413 &route_ctx(ConnectionId::new(92)).0,
9414 route_open_frame_with_admission_facts(
9415 21,
9416 "target",
9417 unique_project_root("admission-facts"),
9418 Some(subc_control::ConsumerIdentity {
9419 module_id: "never-spawned".to_string(),
9420 launch_nonce: "any-nonce".to_string(),
9421 }),
9422 None,
9423 ),
9424 )
9425 .await
9426 .unwrap();
9427 assert_eq!(
9428 parse_error(&never_spawned[0])["code"],
9429 "bad_consumer_identity",
9430 "an unspawned module_id must be refused rather than accepted for lack of a record"
9431 );
9432 }
9433
9434 #[tokio::test]
9457 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
9458 let registry = Arc::new(Registry::default());
9459 let forwarding = Arc::new(ForwardingTable::default());
9460 let supervisor = SupervisorHandle::new();
9461 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9462 let handler =
9463 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9464 .with_supervisor(supervisor);
9465
9466 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
9467 hello_via_sink(
9468 &handler,
9469 &target_ctx,
9470 &mut target_rx,
9471 hello_frame("target", PROTOCOL_VERSION, 1),
9472 )
9473 .await;
9474
9475 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
9476 let route_handler = handler.clone();
9477 let route_task = tokio::spawn(async move {
9478 route_handler
9479 .handle_control_frame(
9480 &client_ctx,
9481 route_open_frame_with_admission_facts(
9482 30,
9483 "target",
9484 unique_project_root("admission-facts"),
9485 Some(subc_control::ConsumerIdentity {
9486 module_id: "fed".to_string(),
9487 launch_nonce: "fed-nonce".to_string(),
9488 }),
9489 None,
9490 ),
9491 )
9492 .await
9493 .unwrap()
9494 });
9495
9496 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9503 .await
9504 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
9505 .expect("module control channel closed before route.bind");
9506 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9507 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9508 panic!("expected route.bind")
9509 };
9510 assert_eq!(
9511 principal,
9512 Some(Principal::Reserved {
9513 module_id: "fed".to_string()
9514 }),
9515 "a correctly attested consumer must be stamped Reserved for its own id"
9516 );
9517
9518 handler
9519 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9520 .await
9521 .unwrap();
9522 assert!(route_task.await.unwrap().is_empty());
9523 assert!(
9524 matches!(
9525 serde_json::from_slice::<ClientControlResponse>(
9526 &client_rx.recv().await.unwrap().body
9527 )
9528 .unwrap(),
9529 ClientControlResponse::RouteOpen { .. }
9530 ),
9531 "the route must actually open, not merely avoid an error"
9532 );
9533 }
9534
9535 #[tokio::test(start_paused = true)]
9536 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
9537 let registry = Arc::new(Registry::default());
9538 let forwarding = Arc::new(ForwardingTable::default());
9539 let supervisor = SupervisorHandle::new();
9540 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9541 let handler =
9542 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9543 .with_supervisor(supervisor);
9544
9545 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
9546 hello_via_sink(
9547 &handler,
9548 &target_ctx,
9549 &mut target_rx,
9550 hello_frame("target", PROTOCOL_VERSION, 1),
9551 )
9552 .await;
9553
9554 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
9555 let direct_handler = handler.clone();
9556 let direct_open = tokio::spawn(async move {
9557 direct_handler
9558 .handle_control_frame(
9559 &direct_ctx,
9560 route_open_frame(2, "target", unique_project_root("route-census-direct")),
9561 )
9562 .await
9563 .unwrap()
9564 });
9565 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9566 .await
9567 .expect("no direct route.bind within 5s")
9568 .expect("target control channel closed before direct route.bind");
9569 handler
9570 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
9571 .await
9572 .unwrap();
9573 assert!(direct_open.await.unwrap().is_empty());
9574 let _ = direct_rx.recv().await.unwrap();
9575
9576 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
9577 let reserved_handler = handler.clone();
9578 let reserved_open = tokio::spawn(async move {
9579 reserved_handler
9580 .handle_control_frame(
9581 &reserved_ctx,
9582 route_open_frame_with_admission_facts(
9583 3,
9584 "target",
9585 unique_project_root("admission-facts"),
9586 Some(ConsumerIdentity {
9587 module_id: "fed".to_string(),
9588 launch_nonce: "fed-nonce".to_string(),
9589 }),
9590 None,
9591 ),
9592 )
9593 .await
9594 .unwrap()
9595 });
9596 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9597 .await
9598 .expect("no reserved route.bind within 5s")
9599 .expect("target control channel closed before reserved route.bind");
9600 handler
9601 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
9602 .await
9603 .unwrap();
9604 assert!(reserved_open.await.unwrap().is_empty());
9605 let _ = reserved_rx.recv().await.unwrap();
9606
9607 forwarding
9608 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
9609 .unwrap();
9610 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
9611 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
9612 module_id: Some("target".to_string()),
9613 })
9614 .unwrap();
9615 let census_frame =
9616 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
9617 let response = handler
9618 .handle_control_frame(&census_ctx, census_frame)
9619 .await
9620 .unwrap()
9621 .pop()
9622 .unwrap();
9623 let actual: Value = serde_json::from_slice(&response.body).unwrap();
9624 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
9625 assert!(matches!(
9626 decoded,
9627 ClientControlResponse::SupervisorRoutes { .. }
9628 ));
9629 let routes = actual["modules"][0]["routes"].as_array().unwrap();
9630 assert_eq!(routes.len(), 2);
9631 assert!(routes.iter().all(|route| route["draining"] == true));
9632 assert!(
9635 routes.iter().all(|route| route["drain_reason"] == "reload"),
9636 "draining routes must name the drain's reason: {routes:?}"
9637 );
9638 assert!(routes.iter().any(|route| {
9639 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
9640 }));
9641 assert!(routes.iter().any(|route| {
9642 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
9643 }));
9644
9645 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9646 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
9647 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9648 std::fs::write(
9649 &golden_path,
9650 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
9651 )
9652 .unwrap();
9653 }
9654 let expected: Value =
9655 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
9656 assert_eq!(actual, expected);
9657 }
9658
9659 async fn query_live_roots(
9660 handler: &ControlHandler,
9661 module_ctx: &RouteCtx,
9662 ) -> ModuleControlResponseToModule {
9663 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
9664 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
9665 let response = handler
9666 .handle_control_frame(module_ctx, frame)
9667 .await
9668 .unwrap()
9669 .pop()
9670 .unwrap();
9671 serde_json::from_slice(&response.body).unwrap()
9672 }
9673
9674 #[tokio::test(start_paused = true)]
9675 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
9676 let registry = Arc::new(Registry::default());
9677 let forwarding = Arc::new(ForwardingTable::default());
9678 let handler = ControlHandler::with_forwarding(registry, forwarding);
9679 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
9680 hello_via_sink(
9681 &handler,
9682 &target_ctx,
9683 &mut target_rx,
9684 hello_frame("target", PROTOCOL_VERSION, 1),
9685 )
9686 .await;
9687 let root = unique_project_root("live-roots-known");
9688 let path = ProjectRootId::from_path_allowing_missing(root.path())
9689 .unwrap()
9690 .as_path()
9691 .to_path_buf();
9692 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
9693 let open_handler = handler.clone();
9694 let opened = tokio::spawn(async move {
9695 open_handler
9696 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
9697 .await
9698 .unwrap()
9699 });
9700 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9701 .await
9702 .unwrap()
9703 .unwrap();
9704 handler
9705 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
9706 .await
9707 .unwrap();
9708 assert!(opened.await.unwrap().is_empty());
9709 let _ = client_rx.recv().await.unwrap();
9710
9711 let root = unique_project_root("live-roots-pending");
9712 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
9713 .unwrap()
9714 .as_path()
9715 .to_path_buf();
9716 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
9717 let open_handler = handler.clone();
9718 let pending = tokio::spawn(async move {
9719 open_handler
9720 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
9721 .await
9722 .unwrap()
9723 });
9724 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9725 .await
9726 .unwrap()
9727 .unwrap();
9728 let actual = query_live_roots(&handler, &target_ctx).await;
9729 let ModuleControlResponseToModule::LiveRoots {
9730 roots,
9731 unknown_root_bindings,
9732 total_bindings,
9733 } = actual
9734 else {
9735 panic!("expected live roots")
9736 };
9737 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
9738 assert_eq!(unknown_root_bindings, 0);
9739 assert_eq!(
9740 roots.len(),
9741 2,
9742 "root-known arm must retain each canonical root"
9743 );
9744 assert_eq!(
9745 total_bindings,
9746 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9747 );
9748 let counts = roots
9749 .iter()
9750 .map(|root| (root.project_root.clone(), root.bound, root.pending))
9751 .collect::<Vec<_>>();
9752 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
9753 expected.sort_by(|a, b| a.0.cmp(&b.0));
9754 assert_eq!(
9755 counts, expected,
9756 "roots must sort by path and count pending separately"
9757 );
9758 handler
9759 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
9760 .await
9761 .unwrap();
9762 assert!(pending.await.unwrap().is_empty());
9763 }
9764
9765 #[tokio::test(start_paused = true)]
9766 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
9767 let forwarding = Arc::new(ForwardingTable::default());
9768 let handler =
9769 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9770 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
9771 hello_via_sink(
9772 &handler,
9773 &target_ctx,
9774 &mut target_rx,
9775 hello_frame("target", PROTOCOL_VERSION, 1),
9776 )
9777 .await;
9778 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
9779 let pending = forwarding
9780 .begin_route_bind_relay_for_test(
9781 client_ctx.connection_id,
9782 client_ctx.egress.clone(),
9783 2,
9784 "target",
9785 )
9786 .unwrap();
9787 forwarding
9788 .complete_pending_relay(
9789 target_ctx.connection_id,
9790 pending.corr,
9791 RouteBindRelayOutcome::Accepted,
9792 )
9793 .unwrap();
9794 let actual = query_live_roots(&handler, &target_ctx).await;
9795 let ModuleControlResponseToModule::LiveRoots {
9796 roots,
9797 unknown_root_bindings,
9798 total_bindings,
9799 } = actual
9800 else {
9801 panic!("expected live roots")
9802 };
9803 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
9804 assert_eq!(
9805 unknown_root_bindings, 1,
9806 "unknown-root arm must not read as no bindings"
9807 );
9808 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
9809 assert_eq!(
9810 total_bindings,
9811 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9812 );
9813 }
9814
9815 #[tokio::test(start_paused = true)]
9823 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
9824 let forwarding = Arc::new(ForwardingTable::default());
9825 let handler =
9826 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9827 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
9828 let replies = handler
9829 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
9830 .await
9831 .unwrap();
9832 let queued_by_hello = module_rx.len();
9833
9834 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
9835 let open_handler = handler.clone();
9836 let open = tokio::spawn(async move {
9837 open_handler
9838 .handle_control_frame(
9839 &client_ctx,
9840 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
9841 )
9842 .await
9843 .unwrap()
9844 });
9845 let mut spins = 0;
9847 while module_rx.len() == queued_by_hello {
9848 spins += 1;
9849 assert!(spins < 10_000, "route.open never queued a route.bind");
9850 tokio::task::yield_now().await;
9851 }
9852
9853 for reply in replies {
9855 module_ctx.egress.send(reply).await.unwrap();
9856 }
9857
9858 let first = module_rx.recv().await.unwrap().frame;
9859 assert_eq!(
9860 first.header.ty,
9861 FrameType::HelloAck,
9862 "the first frame a registering module reads must be its HELLO_ACK"
9863 );
9864 assert_eq!(first.header.corr, 7);
9865 let second = module_rx.recv().await.unwrap().frame;
9866 assert_eq!(second.header.ty, FrameType::Request);
9867 assert!(
9868 matches!(
9869 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
9870 ModuleControlRequest::RouteBind { .. }
9871 ),
9872 "the route.bind follows the ack"
9873 );
9874 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
9875
9876 handler
9877 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
9878 .await
9879 .unwrap();
9880 assert!(open.await.unwrap().is_empty());
9881 let _ = client_rx.recv().await.unwrap();
9882 }
9883
9884 #[tokio::test(start_paused = true)]
9885 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
9886 let handler = ControlHandler::with_forwarding(
9887 Arc::new(Registry::default()),
9888 Arc::new(ForwardingTable::default()),
9889 );
9890 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
9891 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
9892 hello_via_sink(
9893 &handler,
9894 &first_ctx,
9895 &mut first_rx,
9896 hello_frame("first", PROTOCOL_VERSION, 1),
9897 )
9898 .await;
9899 hello_via_sink(
9900 &handler,
9901 &second_ctx,
9902 &mut second_rx,
9903 hello_frame("second", PROTOCOL_VERSION, 2),
9904 )
9905 .await;
9906 let root = unique_project_root("second-only");
9907 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
9908 let cloned = handler.clone();
9909 let open = tokio::spawn(async move {
9910 cloned
9911 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
9912 .await
9913 .unwrap()
9914 });
9915 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
9916 .await
9917 .unwrap()
9918 .unwrap();
9919 let first = query_live_roots(&handler, &first_ctx).await;
9920 let second = query_live_roots(&handler, &second_ctx).await;
9921 assert!(
9922 matches!(
9923 first,
9924 ModuleControlResponseToModule::LiveRoots {
9925 total_bindings: 0,
9926 ..
9927 }
9928 ),
9929 "cross-module scope must not expose another module's roots"
9930 );
9931 assert!(
9932 matches!(
9933 second,
9934 ModuleControlResponseToModule::LiveRoots {
9935 total_bindings: 1,
9936 ..
9937 }
9938 ),
9939 "second module must see its pending route"
9940 );
9941 handler
9942 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
9943 .await
9944 .unwrap();
9945 assert!(open.await.unwrap().is_empty());
9946 }
9947
9948 #[tokio::test(start_paused = true)]
9949 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
9950 let handler = ControlHandler::with_forwarding(
9951 Arc::new(Registry::default()),
9952 Arc::new(ForwardingTable::default()),
9953 );
9954 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
9955 hello_via_sink(
9956 &handler,
9957 &target_ctx,
9958 &mut target_rx,
9959 hello_frame("target", PROTOCOL_VERSION, 1),
9960 )
9961 .await;
9962 let actual = query_live_roots(&handler, &target_ctx).await;
9963 let ModuleControlResponseToModule::LiveRoots {
9964 roots,
9965 unknown_root_bindings,
9966 total_bindings,
9967 } = actual
9968 else {
9969 panic!("expected live roots")
9970 };
9971 assert!(roots.is_empty());
9972 assert_eq!(unknown_root_bindings, 0);
9973 assert_eq!(total_bindings, 0);
9974 assert_eq!(
9975 total_bindings,
9976 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9977 );
9978 }
9979
9980 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
9987 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9988 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
9989 let text = std::fs::read_to_string(&path)
9990 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
9991 let vectors: Vec<(String, Value)> = text
9992 .lines()
9993 .filter(|line| !line.trim().is_empty())
9994 .map(|line| {
9995 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
9996 let id = entry["corpus_id"]
9997 .as_str()
9998 .expect("every vector carries a corpus_id")
9999 .to_string();
10000 (id, entry["package"].clone())
10001 })
10002 .collect();
10003 assert_eq!(
10007 vectors.len(),
10008 3,
10009 "vendored fed corpus changed size; re-sync from subc-federation"
10010 );
10011
10012 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
10027 let richest = vectors
10028 .iter()
10029 .filter_map(|(_, package)| package.as_object())
10030 .map(|object| {
10031 object
10032 .keys()
10033 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
10034 .count()
10035 })
10036 .max()
10037 .unwrap_or(0);
10038 assert!(
10039 richest >= 2,
10040 "vendored corpus no longer carries a package with unmodelled fields, \
10041 so the relay test can no longer distinguish a verbatim relay from a lossy one"
10042 );
10043
10044 vectors
10045 }
10046
10047 #[tokio::test]
10055 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
10056 for (corpus_id, package) in fed_admission_facts_vectors() {
10057 let registry = Arc::new(Registry::default());
10058 let forwarding = Arc::new(ForwardingTable::default());
10059 let supervisor = SupervisorHandle::new();
10060 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10061 let handler =
10062 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10063 .with_supervisor(supervisor)
10064 .with_admission_facts_config(
10065 Some("fed".to_string()),
10066 Some(vec!["target".to_string()]),
10067 );
10068
10069 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10070 hello_via_sink(
10071 &handler,
10072 &target_ctx,
10073 &mut target_rx,
10074 hello_frame("target", PROTOCOL_VERSION, 1),
10075 )
10076 .await;
10077
10078 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
10079 let route_handler = handler.clone();
10080 let expected = package.clone();
10081 let route_task = tokio::spawn(async move {
10082 route_handler
10083 .handle_control_frame(
10084 &client_ctx,
10085 route_open_frame_with_admission_facts(
10086 20,
10087 "target",
10088 unique_project_root("admission-facts"),
10089 Some(subc_control::ConsumerIdentity {
10090 module_id: "fed".to_string(),
10091 launch_nonce: "fed-nonce".to_string(),
10092 }),
10093 Some(package),
10094 ),
10095 )
10096 .await
10097 .unwrap()
10098 });
10099
10100 let bind_frame = target_rx.recv().await.unwrap();
10101 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10102 let ModuleControlRequest::RouteBind {
10103 admission_facts, ..
10104 } = bind
10105 else {
10106 panic!("{corpus_id}: expected route.bind")
10107 };
10108 assert_eq!(
10109 admission_facts,
10110 Some(expected),
10111 "{corpus_id}: relay must not add, drop or reshape any field"
10112 );
10113
10114 handler
10115 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10116 .await
10117 .unwrap();
10118 route_task.await.unwrap();
10119 }
10120 }
10121
10122 #[tokio::test]
10123 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
10124 let registry = Arc::new(Registry::default());
10125 let forwarding = Arc::new(ForwardingTable::default());
10126 let supervisor = SupervisorHandle::new();
10127 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10128 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
10129 let handler =
10130 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10131 .with_supervisor(supervisor)
10132 .with_admission_facts_config(
10133 Some("fed".to_string()),
10134 Some(vec!["target".to_string()]),
10135 );
10136
10137 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
10138 hello_via_sink(
10139 &handler,
10140 &target_ctx,
10141 &mut target_rx,
10142 hello_frame("target", PROTOCOL_VERSION, 1),
10143 )
10144 .await;
10145 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
10146 hello_via_sink(
10147 &handler,
10148 &other_ctx,
10149 &mut other_rx,
10150 hello_frame("other", PROTOCOL_VERSION, 2),
10151 )
10152 .await;
10153
10154 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
10155 let expected_facts = facts.clone();
10156 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
10157 let route_handler = handler.clone();
10158 let route_task = tokio::spawn(async move {
10159 route_handler
10160 .handle_control_frame(
10161 &client_ctx,
10162 route_open_frame_with_admission_facts(
10163 10,
10164 "target",
10165 unique_project_root("admission-facts"),
10166 Some(subc_control::ConsumerIdentity {
10167 module_id: "fed".to_string(),
10168 launch_nonce: "fed-nonce".to_string(),
10169 }),
10170 Some(facts.clone()),
10171 ),
10172 )
10173 .await
10174 .unwrap()
10175 });
10176 let bind_frame = target_rx.recv().await.unwrap();
10177 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10178 let ModuleControlRequest::RouteBind {
10179 admission_facts, ..
10180 } = bind
10181 else {
10182 panic!("expected route.bind")
10183 };
10184 assert_eq!(admission_facts, Some(expected_facts));
10185 handler
10186 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10187 .await
10188 .unwrap();
10189 assert!(route_task.await.unwrap().is_empty());
10190 assert!(matches!(
10191 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
10192 .unwrap(),
10193 ClientControlResponse::RouteOpen { .. }
10194 ));
10195
10196 let direct = handler
10197 .handle_control_frame(
10198 &route_ctx(ConnectionId::new(73)).0,
10199 route_open_frame_with_admission_facts(
10200 11,
10201 "target",
10202 unique_project_root("admission-facts"),
10203 None,
10204 Some(json!({"x": 1})),
10205 ),
10206 )
10207 .await
10208 .unwrap();
10209 assert_eq!(
10210 parse_error(&direct[0])["code"],
10211 "admission_facts_not_permitted"
10212 );
10213
10214 let different_reserved = handler
10215 .handle_control_frame(
10216 &route_ctx(ConnectionId::new(77)).0,
10217 route_open_frame_with_admission_facts(
10218 15,
10219 "target",
10220 unique_project_root("admission-facts"),
10221 Some(subc_control::ConsumerIdentity {
10222 module_id: "other".to_string(),
10223 launch_nonce: "other-nonce".to_string(),
10224 }),
10225 Some(json!({"x": 1})),
10226 ),
10227 )
10228 .await
10229 .unwrap();
10230 assert_eq!(
10231 parse_error(&different_reserved[0])["code"],
10232 "admission_facts_not_permitted"
10233 );
10234
10235 let other_target = handler
10236 .handle_control_frame(
10237 &route_ctx(ConnectionId::new(74)).0,
10238 route_open_frame_with_admission_facts(
10239 12,
10240 "other",
10241 unique_project_root("admission-facts"),
10242 Some(subc_control::ConsumerIdentity {
10243 module_id: "fed".to_string(),
10244 launch_nonce: "fed-nonce".to_string(),
10245 }),
10246 Some(json!({"x": 1})),
10247 ),
10248 )
10249 .await
10250 .unwrap();
10251 assert_eq!(
10252 parse_error(&other_target[0])["code"],
10253 "admission_facts_target_not_allowed"
10254 );
10255
10256 let nonexistent = handler
10257 .handle_control_frame(
10258 &route_ctx(ConnectionId::new(75)).0,
10259 route_open_frame_with_admission_facts(
10260 13,
10261 "missing",
10262 unique_project_root("admission-facts"),
10263 None,
10264 Some(json!({"x": 1})),
10265 ),
10266 )
10267 .await
10268 .unwrap();
10269 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
10270
10271 let described = handler
10272 .handle_control_frame(
10273 &route_ctx(ConnectionId::new(76)).0,
10274 Frame::build(
10275 FrameType::Request,
10276 control_flags(),
10277 0,
10278 0,
10279 14,
10280 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10281 )
10282 .unwrap(),
10283 )
10284 .await
10285 .unwrap();
10286 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10287 serde_json::from_slice(&described[0].body).unwrap()
10288 else {
10289 panic!("expected server.describe response")
10290 };
10291 assert!(capabilities
10292 .iter()
10293 .any(|cap| cap == "admission_facts_relay_v1"));
10294 }
10295
10296 #[tokio::test]
10297 async fn admission_facts_without_configured_carrier_are_rejected() {
10298 let registry = Arc::new(Registry::default());
10299 let forwarding = Arc::new(ForwardingTable::default());
10300 let handler = ControlHandler::with_forwarding(registry, forwarding);
10301 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
10302 hello_via_sink(
10303 &handler,
10304 &target_ctx,
10305 &mut target_rx,
10306 hello_frame("target", PROTOCOL_VERSION, 1),
10307 )
10308 .await;
10309
10310 let responses = handler
10311 .handle_control_frame(
10312 &route_ctx(ConnectionId::new(79)).0,
10313 route_open_frame_with_admission_facts(
10314 16,
10315 "target",
10316 unique_project_root("admission-facts"),
10317 None,
10318 Some(json!({"x": 1})),
10319 ),
10320 )
10321 .await
10322 .unwrap();
10323 assert_eq!(
10324 parse_error(&responses[0])["code"],
10325 "admission_facts_not_permitted"
10326 );
10327 }
10328
10329 #[tokio::test]
10330 async fn route_open_relays_consumer_capabilities_verbatim() {
10331 let registry = Arc::new(Registry::default());
10332 let forwarding = Arc::new(ForwardingTable::default());
10333 let handler =
10334 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10335 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
10336 hello_via_sink(
10337 &handler,
10338 &module_ctx,
10339 &mut module_rx,
10340 hello_frame("aft", PROTOCOL_VERSION, 7),
10341 )
10342 .await;
10343
10344 let expected = vec!["elicitation".to_string(), "roots".to_string()];
10345 let expected_for_request = expected.clone();
10346 let project_root = unique_project_root("consumer-capabilities-present");
10347 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
10348 let route_handler = handler.clone();
10349 let route_task = tokio::spawn(async move {
10350 route_handler
10351 .handle_control_frame(
10352 &client_ctx,
10353 route_open_frame_with_consumer_capabilities(
10354 401,
10355 "aft",
10356 project_root,
10357 Some(expected_for_request),
10358 ),
10359 )
10360 .await
10361 .unwrap()
10362 });
10363 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10364 .await
10365 .unwrap()
10366 .unwrap();
10367 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10368 let ModuleControlRequest::RouteBind {
10369 consumer_capabilities,
10370 ..
10371 } = bind
10372 else {
10373 panic!("expected route.bind request, got {bind:?}");
10374 };
10375 assert_eq!(consumer_capabilities, Some(expected.clone()));
10376
10377 handler
10378 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10379 .await
10380 .unwrap();
10381 let route_response = route_task.await.unwrap();
10382 assert!(route_response.is_empty());
10383 let published = client_rx.recv().await.unwrap();
10384 assert!(matches!(
10385 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10386 ClientControlResponse::RouteOpen { .. }
10387 ));
10388 }
10389
10390 #[tokio::test]
10391 async fn route_open_without_consumer_capabilities_relays_none() {
10392 let registry = Arc::new(Registry::default());
10393 let forwarding = Arc::new(ForwardingTable::default());
10394 let handler =
10395 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10396 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
10397 hello_via_sink(
10398 &handler,
10399 &module_ctx,
10400 &mut module_rx,
10401 hello_frame("aft", PROTOCOL_VERSION, 7),
10402 )
10403 .await;
10404
10405 let project_root = unique_project_root("consumer-capabilities-absent");
10406 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
10407 let route_handler = handler.clone();
10408 let route_task = tokio::spawn(async move {
10409 route_handler
10410 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
10411 .await
10412 .unwrap()
10413 });
10414 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10415 .await
10416 .unwrap()
10417 .unwrap();
10418 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10419 let ModuleControlRequest::RouteBind {
10420 consumer_capabilities,
10421 ..
10422 } = bind
10423 else {
10424 panic!("expected route.bind request, got {bind:?}");
10425 };
10426 assert_eq!(consumer_capabilities, None);
10427
10428 handler
10429 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10430 .await
10431 .unwrap();
10432 let route_response = route_task.await.unwrap();
10433 assert!(route_response.is_empty());
10434 let published = client_rx.recv().await.unwrap();
10435 assert!(matches!(
10436 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10437 ClientControlResponse::RouteOpen { .. }
10438 ));
10439 }
10440
10441 async fn bind_role_versions_for(
10445 sent: Option<BTreeMap<String, String>>,
10446 connection: u64,
10447 ) -> Option<BTreeMap<String, String>> {
10448 let registry = Arc::new(Registry::default());
10449 let forwarding = Arc::new(ForwardingTable::default());
10450 let handler =
10451 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10452 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
10453 hello_via_sink(
10454 &handler,
10455 &module_ctx,
10456 &mut module_rx,
10457 hello_frame("aft", PROTOCOL_VERSION, 7),
10458 )
10459 .await;
10460 let project_root = unique_project_root("role-versions");
10461 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
10462 let route_handler = handler.clone();
10463 let route_task = tokio::spawn(async move {
10464 route_handler
10465 .handle_control_frame(
10466 &client_ctx,
10467 route_open_frame_with_role_versions(403, "aft", project_root, sent),
10468 )
10469 .await
10470 .unwrap()
10471 });
10472 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10473 .await
10474 .expect("a well-formed route.open reaches the module as a bind")
10475 .unwrap();
10476 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10477 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
10478 panic!("expected route.bind request, got {bind:?}");
10479 };
10480 handler
10481 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10482 .await
10483 .unwrap();
10484 assert!(route_task.await.unwrap().is_empty());
10485 let published = client_rx.recv().await.unwrap();
10486 assert!(matches!(
10487 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10488 ClientControlResponse::RouteOpen { .. }
10489 ));
10490 role_versions
10491 }
10492
10493 #[tokio::test]
10494 async fn route_open_relays_role_versions_verbatim() {
10495 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
10496 assert_eq!(
10497 bind_role_versions_for(Some(sent.clone()), 141).await,
10498 Some(sent)
10499 );
10500 }
10501
10502 #[tokio::test]
10505 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
10506 assert_eq!(bind_role_versions_for(None, 143).await, None);
10507 assert_eq!(
10508 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
10509 None
10510 );
10511 }
10512
10513 #[tokio::test]
10514 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
10515 let registry = Arc::new(Registry::default());
10516 let forwarding = Arc::new(ForwardingTable::default());
10517 let handler =
10518 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10519 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
10520 hello_via_sink(
10521 &handler,
10522 &module_ctx,
10523 &mut module_rx,
10524 hello_frame("aft", PROTOCOL_VERSION, 7),
10525 )
10526 .await;
10527
10528 let nine: BTreeMap<String, String> = (0..9)
10529 .map(|index| (format!("role-{index}"), "v1".to_string()))
10530 .collect();
10531 for (label, malformed) in [
10532 (
10533 "invalid role name",
10534 role_versions(&[("Tool_Provider", "v1")]),
10535 ),
10536 ("invalid version", role_versions(&[("tool-provider", "v0")])),
10537 ("nine entries", nine),
10538 ] {
10539 let responses = tokio::time::timeout(
10542 Duration::from_secs(1),
10543 handler.handle_control_frame(
10544 &route_ctx(ConnectionId::new(148)).0,
10545 route_open_frame_with_role_versions(
10546 404,
10547 "aft",
10548 unique_project_root("role-versions-malformed"),
10549 Some(malformed),
10550 ),
10551 ),
10552 )
10553 .await
10554 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
10555 .unwrap();
10556 assert_eq!(responses.len(), 1, "{label}");
10557 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
10558 let error = parse_error(&responses[0]);
10559 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
10560 assert_eq!(
10561 error["detail"]["field"], "role_versions",
10562 "{label}: {error}"
10563 );
10564 assert!(
10565 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
10566 "{label}: a malformed declaration is terminal"
10567 );
10568 assert!(
10569 module_rx.try_recv().is_err(),
10570 "{label}: the module must never see a bind"
10571 );
10572 }
10573 }
10574
10575 #[tokio::test]
10579 async fn route_role_versions_capability_is_advertised() {
10580 let handler = ControlHandler::new(Arc::new(Registry::default()));
10581 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
10582 let ack = hello_via_sink(
10583 &handler,
10584 &ctx,
10585 &mut rx,
10586 hello_frame("m", PROTOCOL_VERSION, 1),
10587 )
10588 .await;
10589 let ack = parse_ack(&ack);
10590 assert!(
10591 ack.subc_capabilities
10592 .iter()
10593 .any(|c| c == "route-role-versions/v1"),
10594 "{:?}",
10595 ack.subc_capabilities
10596 );
10597
10598 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
10599 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
10600 let reply = handler
10601 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
10602 .await
10603 .unwrap()
10604 .pop()
10605 .unwrap();
10606 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10607 serde_json::from_slice(&reply.body).unwrap()
10608 else {
10609 panic!("not a server.describe reply");
10610 };
10611 assert!(
10612 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
10613 "{capabilities:?}"
10614 );
10615 }
10616
10617 #[tokio::test]
10621 async fn catalog_events_capability_is_advertised() {
10622 let handler = ControlHandler::new(Arc::new(Registry::default()));
10623 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
10624 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
10625 let reply = handler
10626 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
10627 .await
10628 .unwrap()
10629 .pop()
10630 .unwrap();
10631 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10632 serde_json::from_slice(&reply.body).unwrap()
10633 else {
10634 panic!("not a server.describe reply");
10635 };
10636 assert!(
10637 capabilities
10638 .iter()
10639 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
10640 "{capabilities:?}"
10641 );
10642 }
10643
10644 #[tokio::test]
10645 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
10646 let registry = Arc::new(Registry::default());
10647 let forwarding = Arc::new(ForwardingTable::default());
10648 let handler =
10649 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10650 .with_health_probe_timeout(Duration::from_secs(5));
10651 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
10652 hello_via_sink(
10653 &handler,
10654 &module_ctx,
10655 &mut module_rx,
10656 non_routable_hello_frame_with_control_ops(
10657 "mcp",
10658 300,
10659 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10660 ),
10661 )
10662 .await;
10663 assert!(registry
10664 .get_module("mcp")
10665 .unwrap()
10666 .unwrap()
10667 .manifest
10668 .provides
10669 .is_empty());
10670
10671 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
10672 let route_response = handler
10673 .handle_control_frame(
10674 &route_client_ctx,
10675 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
10676 )
10677 .await
10678 .unwrap();
10679 assert_eq!(route_response[0].header.ty, FrameType::Error);
10680 assert_eq!(
10681 parse_error(&route_response[0])["code"],
10682 "target_unavailable"
10683 );
10684 assert!(parse_error(&route_response[0])["message"]
10685 .as_str()
10686 .unwrap()
10687 .contains("does not provide the requested target"));
10688 assert!(module_rx.try_recv().is_err());
10689
10690 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
10691 let health_handler = handler.clone();
10692 let health_task = tokio::spawn(async move {
10693 health_handler
10694 .handle_control_frame(
10695 &health_client_ctx,
10696 supervisor_health_probe_frame(302, "mcp"),
10697 )
10698 .await
10699 .unwrap()
10700 });
10701 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10702 .await
10703 .unwrap()
10704 .unwrap();
10705 assert_eq!(
10706 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10707 ModuleControlRequest::HealthCheck {}
10708 );
10709 handler
10710 .handle_control_frame(
10711 &module_ctx,
10712 health_response(health_frame.header.corr, HealthStatus::Ok),
10713 )
10714 .await
10715 .unwrap();
10716 let health_response = health_task.await.unwrap();
10717 assert_eq!(health_response[0].header.ty, FrameType::Response);
10718 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10719 ClientControlResponse::SupervisorHealthProbe {
10720 module_id, status, ..
10721 } => {
10722 assert_eq!(module_id, "mcp");
10723 assert_eq!(status, HealthStatus::Ok);
10724 }
10725 other => panic!("unexpected health response: {other:?}"),
10726 }
10727
10728 forwarding
10733 .cleanup_connection(module_ctx.connection_id)
10734 .unwrap();
10735 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
10736 let cleanup_response = tokio::time::timeout(
10737 Duration::from_millis(200),
10738 handler.handle_control_frame(
10739 &cleanup_probe_ctx,
10740 supervisor_health_probe_frame(303, "mcp"),
10741 ),
10742 )
10743 .await
10744 .expect("probe should fail immediately when the control lane is gone")
10745 .unwrap();
10746 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
10747 assert_eq!(
10748 parse_error(&cleanup_response[0])["code"],
10749 "target_unavailable"
10750 );
10751 assert!(parse_error(&cleanup_response[0])["message"]
10752 .as_str()
10753 .unwrap()
10754 .contains("no module connection"));
10755
10756 handler
10757 .cleanup_connection(module_ctx.connection_id)
10758 .unwrap();
10759 }
10760
10761 #[tokio::test]
10762 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
10763 let registry = Arc::new(Registry::default());
10764 let supervisor_handle = SupervisorHandle::new();
10765 let supervisor =
10766 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10767 .with_handle(supervisor_handle.clone())
10768 .with_connection_file_path(
10769 std::env::temp_dir()
10770 .join(format!("subc-route-open-warming-{}", std::process::id())),
10771 );
10772 let module = supervisor
10773 .supervise_configured(
10774 ModuleSpec {
10775 module_id: "warming".to_string(),
10776 program: fake_aft_stub_path(),
10777 args: Vec::new(),
10778 env: Vec::new(),
10779 reserved: false,
10780 reserved_prefixes: Vec::new(),
10781 protocol: ModuleProtocol::Subc,
10782 overlap: Default::default(),
10783 },
10784 true,
10785 )
10786 .unwrap();
10787 assert_eq!(module.state().unwrap(), ModuleState::Running);
10788
10789 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10790 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
10791 let response = handler
10792 .handle_control_frame(
10793 &ctx,
10794 route_open_frame(304, "warming", unique_project_root("warming")),
10795 )
10796 .await
10797 .unwrap();
10798 module.stop().await.unwrap();
10799
10800 assert_eq!(response[0].header.ty, FrameType::Error);
10801 let error = parse_error(&response[0]);
10802 assert_eq!(error["code"], "module_warming");
10803 assert!(error["message"]
10804 .as_str()
10805 .unwrap()
10806 .contains("state=running, enabled=true, live=false"));
10807 }
10808
10809 #[test]
10810 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
10811 let handler = ControlHandler::new(Arc::new(Registry::default()));
10812 let capture = EventCapture::default();
10813 let _subscriber =
10814 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10815 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
10816 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
10817 let pending = (0..limit).collect::<Vec<_>>();
10818 let response = handler
10819 .route_open_capacity_refusal(
10820 &ctx,
10821 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
10822 "busy",
10823 pending.len(),
10824 limit,
10825 )
10826 .unwrap();
10827 assert_eq!(parse_error(&response)["code"], "target_unavailable");
10828 let event = capture
10829 .events()
10830 .into_iter()
10831 .find(|event| {
10832 event.target == "control"
10833 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
10834 })
10835 .expect("connection admission refusal event");
10836 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10837 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10838 }
10839
10840 #[test]
10841 fn route_open_target_cap_logs_admission_reason_and_capacity() {
10842 let handler = ControlHandler::new(Arc::new(Registry::default()));
10843 let capture = EventCapture::default();
10844 let _subscriber =
10845 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10846 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
10847 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
10848 let guards = (0..limit)
10849 .map(|_| {
10850 handler
10851 .route_bind_concurrency
10852 .try_admit("busy", limit)
10853 .unwrap()
10854 })
10855 .collect::<Vec<_>>();
10856 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
10857 Err(in_flight) => in_flight,
10858 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
10859 };
10860 let response = handler
10861 .route_open_target_capacity_refusal(
10862 &ctx,
10863 &route_open_frame(397, "busy", unique_project_root("target-cap")),
10864 "busy",
10865 in_flight,
10866 )
10867 .unwrap();
10868 assert_eq!(parse_error(&response)["code"], "target_unavailable");
10869 let event = capture
10870 .events()
10871 .into_iter()
10872 .find(|event| {
10873 event.target == "control"
10874 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
10875 })
10876 .expect("target admission refusal event");
10877 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10878 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10879 drop(guards);
10880 }
10881
10882 #[tokio::test(flavor = "current_thread")]
10883 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
10884 let handler = ControlHandler::new(Arc::new(Registry::default()));
10885 let capture = EventCapture::default();
10886 let _subscriber =
10887 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10888 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
10889 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10890
10891 let _ = handler
10892 .handle_client_control_request(
10893 &ctx,
10894 frame,
10895 ClientControlRequest::SupervisorSetEnabled {
10896 module_id: "broca".to_string(),
10897 enabled: false,
10898 },
10899 )
10900 .await
10901 .unwrap();
10902
10903 let events = capture
10904 .events()
10905 .into_iter()
10906 .filter(|event| {
10907 event.target == "control"
10908 && event.fields.get("message").map(String::as_str)
10909 == Some("supervisor request received")
10910 })
10911 .collect::<Vec<_>>();
10912 assert_eq!(events.len(), 1, "one supervisor request log line");
10913 let event = &events[0];
10914 assert_eq!(event.level, tracing::Level::INFO);
10915 assert_eq!(
10916 event.fields.get("op"),
10917 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
10918 );
10919 assert_eq!(
10920 event.fields.get("module_id"),
10921 Some(&"\"broca\"".to_string())
10922 );
10923 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
10924 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
10925 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
10926 }
10927
10928 #[tokio::test(flavor = "current_thread")]
10929 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
10930 let registry = Arc::new(Registry::default());
10931 let module_connection = ConnectionId::new(303);
10932 registry
10933 .register_with_control_ops(
10934 manifest("aft", PROTOCOL_VERSION),
10935 PROTOCOL_VERSION,
10936 module_connection,
10937 module_baseline_control_ops(),
10938 )
10939 .unwrap();
10940 let handler = ControlHandler::new(registry);
10941 let capture = EventCapture::default();
10942 let _subscriber =
10943 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10944 let (ctx, _rx) = route_ctx(module_connection);
10945 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10946
10947 let _ = handler
10948 .handle_client_control_request(
10949 &ctx,
10950 frame,
10951 ClientControlRequest::SupervisorSetEnabled {
10952 module_id: "broca".to_string(),
10953 enabled: false,
10954 },
10955 )
10956 .await
10957 .unwrap();
10958
10959 let callers = capture
10960 .events()
10961 .into_iter()
10962 .filter(|event| {
10963 event.fields.get("message").map(String::as_str)
10964 == Some("supervisor request received")
10965 })
10966 .map(|event| event.fields.get("caller").cloned())
10967 .collect::<Vec<_>>();
10968 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
10969 }
10970
10971 #[tokio::test(flavor = "current_thread")]
10972 async fn supervisor_list_does_not_log_request_received() {
10973 let handler = ControlHandler::new(Arc::new(Registry::default()));
10974 let capture = EventCapture::default();
10975 let _subscriber =
10976 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10977 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
10978 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
10979
10980 handler
10981 .handle_client_control_request(&ctx, frame, ClientControlRequest::SupervisorList {})
10982 .await
10983 .unwrap();
10984
10985 assert!(capture.events().into_iter().all(|event| {
10986 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10987 }));
10988 }
10989
10990 #[tokio::test(flavor = "current_thread")]
10991 async fn supervisor_rescan_preview_does_not_log_request_received() {
10992 let handler = ControlHandler::new(Arc::new(Registry::default()));
10993 let capture = EventCapture::default();
10994 let _subscriber =
10995 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10996 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
10997 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
10998
10999 handler
11000 .handle_client_control_request(
11001 &ctx,
11002 frame,
11003 ClientControlRequest::SupervisorRescan { preview: true },
11004 )
11005 .await
11006 .unwrap();
11007
11008 assert!(capture.events().into_iter().all(|event| {
11009 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11010 }));
11011 }
11012
11013 #[tokio::test]
11018 async fn route_open_refusal_names_the_check_that_refused() {
11019 let handler = ControlHandler::new(Arc::new(Registry::default()));
11020 let capture = EventCapture::default();
11021 let _subscriber =
11022 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11023 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11024 let response = handler
11025 .handle_control_frame(
11026 &ctx,
11027 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
11028 )
11029 .await
11030 .unwrap();
11031
11032 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11033 let event = capture
11034 .events()
11035 .into_iter()
11036 .find(|event| {
11037 event.target == "control"
11038 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11039 })
11040 .expect("route.open refusal event");
11041 assert_eq!(
11042 event.fields.get("reason"),
11043 Some(&"\"not_registered\"".to_string())
11044 );
11045 }
11046
11047 #[tokio::test]
11048 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
11049 let registry = Arc::new(Registry::default());
11050 let supervisor_handle = SupervisorHandle::new();
11051 let supervisor =
11052 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11053 .with_handle(supervisor_handle.clone())
11054 .with_connection_file_path(std::env::temp_dir().join(format!(
11055 "subc-route-open-refusal-info-{}",
11056 std::process::id()
11057 )));
11058 let module = supervisor
11059 .supervise_configured(
11060 ModuleSpec {
11061 module_id: "warming".to_string(),
11062 program: fake_aft_stub_path(),
11063 args: Vec::new(),
11064 env: Vec::new(),
11065 reserved: false,
11066 reserved_prefixes: Vec::new(),
11067 protocol: ModuleProtocol::Subc,
11068 overlap: Default::default(),
11069 },
11070 true,
11071 )
11072 .unwrap();
11073 assert_eq!(module.state().unwrap(), ModuleState::Running);
11074
11075 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11076 assert!(handler
11077 .counters()
11078 .snapshot()
11079 .get("route_open_refused_by_code")
11080 .is_none());
11081 let capture = EventCapture::default();
11082 let _subscriber =
11083 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11084 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
11085 let response = handler
11086 .handle_control_frame(
11087 &ctx,
11088 route_open_frame(394, "warming", unique_project_root("refusal-info")),
11089 )
11090 .await
11091 .unwrap();
11092 module.stop().await.unwrap();
11093
11094 assert_eq!(parse_error(&response[0])["code"], "module_warming");
11095 let event = capture
11096 .events()
11097 .into_iter()
11098 .find(|event| {
11099 event.target == "control"
11100 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
11101 })
11102 .expect("route.open refusal event");
11103 assert_eq!(
11104 event.fields.get("module_id"),
11105 Some(&"\"warming\"".to_string())
11106 );
11107 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
11108 assert_eq!(
11109 event.fields.get("reason"),
11110 Some(&"\"supervised_not_registered\"".to_string())
11111 );
11112 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
11113 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
11114 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
11115 assert_eq!(
11116 handler.counters().snapshot()["route_open_refused_by_code"],
11117 json!({ "module_warming": 1 })
11118 );
11119 }
11120
11121 const OUTAGE_START: &str = "route.open refusing module: not serving";
11122 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
11123
11124 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
11125 capture
11126 .events()
11127 .into_iter()
11128 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
11129 .collect()
11130 }
11131
11132 fn supervise_stub(
11133 registry: &Arc<Registry>,
11134 module_id: &str,
11135 enabled: bool,
11136 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
11137 let supervisor_handle = SupervisorHandle::new();
11138 let supervisor =
11139 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
11140 .with_handle(supervisor_handle.clone())
11141 .with_connection_file_path(std::env::temp_dir().join(format!(
11142 "subc-route-outage-{module_id}-{}",
11143 std::process::id()
11144 )));
11145 let module = supervisor
11146 .supervise_configured(
11147 ModuleSpec {
11148 module_id: module_id.to_string(),
11149 program: fake_aft_stub_path(),
11150 args: Vec::new(),
11151 env: Vec::new(),
11152 reserved: false,
11153 reserved_prefixes: Vec::new(),
11154 protocol: ModuleProtocol::Subc,
11155 overlap: Default::default(),
11156 },
11157 enabled,
11158 )
11159 .unwrap();
11160 (supervisor_handle, module)
11161 }
11162
11163 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
11164 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
11165 module_id: module_id.to_string(),
11166 drain_timeout_ms: Some(50),
11167 })
11168 .unwrap();
11169 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11170 }
11171
11172 #[test]
11176 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
11177 let registry = Arc::new(Registry::default());
11178 let forwarding = Arc::new(ForwardingTable::default());
11179 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11180 let second = ControlHandler::with_forwarding(registry, forwarding);
11181 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
11182 }
11183
11184 #[tokio::test(flavor = "current_thread")]
11188 async fn route_open_unknown_module_refusals_add_no_outage_state() {
11189 let handler = ControlHandler::new(Arc::new(Registry::default()));
11190 let capture = EventCapture::default();
11191 let _subscriber =
11192 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11193 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
11194 for corr in 0..8 {
11195 let response = handler
11196 .handle_control_frame(
11197 &ctx,
11198 route_open_frame(
11199 380 + corr,
11200 &format!("nobody-{corr}"),
11201 unique_project_root("outage-unknown"),
11202 ),
11203 )
11204 .await
11205 .unwrap();
11206 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11207 }
11208
11209 assert_eq!(handler.route_outages.tracked_module_count(), 0);
11210 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
11211 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
11212 }
11213
11214 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11219 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
11220 let registry = Arc::new(Registry::default());
11221 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
11222 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11223 let capture = EventCapture::default();
11224 let _subscriber =
11225 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11226 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
11227 handler.route_outages.record_accepted("outage-restart");
11230
11231 let response = handler
11232 .handle_control_frame(
11233 &ctx,
11234 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
11235 )
11236 .await
11237 .unwrap();
11238 assert_eq!(response[0].header.ty, FrameType::Error);
11239 let starts = outage_lines(&capture, OUTAGE_START);
11240 assert_eq!(starts.len(), 1, "{starts:?}");
11241 assert_eq!(starts[0].level, tracing::Level::WARN);
11242 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
11243 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
11244 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
11245 handler.route_outages.record_accepted("outage-restart");
11246 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
11247
11248 let restart = handler
11249 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
11250 .await
11251 .unwrap();
11252 assert_eq!(
11253 restart[0].header.ty,
11254 FrameType::Response,
11255 "{:?}",
11256 parse_error(&restart[0])
11257 );
11258 handler
11259 .handle_control_frame(
11260 &ctx,
11261 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
11262 )
11263 .await
11264 .unwrap();
11265 module.stop().await.unwrap();
11266
11267 let starts = outage_lines(&capture, OUTAGE_START);
11268 assert_eq!(starts.len(), 2, "{starts:?}");
11269 assert_eq!(starts[1].level, tracing::Level::INFO);
11270 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
11271 }
11272
11273 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11277 async fn failed_operator_restart_leaves_no_operator_mark() {
11278 let registry = Arc::new(Registry::default());
11279 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
11280 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11281 let capture = EventCapture::default();
11282 let _subscriber =
11283 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11284 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
11285 handler.route_outages.record_accepted("outage-disabled");
11286
11287 let restart = handler
11288 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
11289 .await
11290 .unwrap();
11291 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
11292 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
11293
11294 handler
11295 .handle_control_frame(
11296 &ctx,
11297 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
11298 )
11299 .await
11300 .unwrap();
11301 let starts = outage_lines(&capture, OUTAGE_START);
11302 assert_eq!(starts.len(), 1, "{starts:?}");
11303 assert_eq!(starts[0].level, tracing::Level::WARN);
11304 }
11305
11306 #[tokio::test(flavor = "current_thread")]
11307 async fn route_open_unknown_module_escapes_target_module_id() {
11308 let handler = ControlHandler::new(Arc::new(Registry::default()));
11309 let capture = EventCapture::default();
11310 let _subscriber =
11311 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11312 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
11313 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11314 let response = handler
11315 .handle_control_frame(
11316 &ctx,
11317 route_open_frame(
11318 395,
11319 hostile_module_id,
11320 unique_project_root("hostile-target-module-id"),
11321 ),
11322 )
11323 .await
11324 .unwrap();
11325
11326 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11327 let event = capture
11328 .events()
11329 .into_iter()
11330 .find(|event| {
11331 event.target == "control"
11332 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11333 })
11334 .expect("route.open unknown-module refusal event");
11335 let logged = event.fields.get("module_id").expect("module_id field");
11336 assert!(!logged.bytes().any(|byte| byte < 0x20));
11337 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
11338 }
11339
11340 #[tokio::test(flavor = "current_thread")]
11341 async fn route_open_module_rejection_uses_daemon_counter_key() {
11342 let registry = Arc::new(Registry::default());
11343 let forwarding = Arc::new(ForwardingTable::default());
11344 let handler =
11345 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11346 let module_connection = ConnectionId::new(95);
11347 let (module_ctx, mut module_rx) = route_ctx(module_connection);
11348 hello_via_sink(
11349 &handler,
11350 &module_ctx,
11351 &mut module_rx,
11352 hello_frame("aft", PROTOCOL_VERSION, 395),
11353 )
11354 .await;
11355
11356 let client_connection = ConnectionId::new(96);
11357 let (client_ctx, _client_rx) = route_ctx(client_connection);
11358 let capture = EventCapture::default();
11359 let _subscriber =
11360 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11361 let (route_task, bind) = relay_route_open(
11362 &handler,
11363 client_connection,
11364 &client_ctx.egress,
11365 &mut module_rx,
11366 396,
11367 "aft",
11368 "hostile-module-code",
11369 )
11370 .await;
11371 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
11372 let rejection = Frame::build(
11373 FrameType::Error,
11374 control_flags(),
11375 0,
11376 0,
11377 bind.header.corr,
11378 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
11379 )
11380 .unwrap();
11381 handler
11382 .handle_control_frame(&module_ctx, rejection)
11383 .await
11384 .unwrap();
11385
11386 let response = route_task.await.unwrap();
11387 assert_eq!(parse_error(&response[0])["code"], hostile_code);
11388 let counters = handler.counters().snapshot();
11389 assert_eq!(
11390 counters["route_open_refused_by_code"],
11391 json!({ "module_rejected": 1 })
11392 );
11393 assert!(counters["route_open_refused_by_code"]
11394 .get(hostile_code)
11395 .is_none());
11396
11397 let event = capture
11398 .events()
11399 .into_iter()
11400 .find(|event| {
11401 event.target == "control"
11402 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
11403 })
11404 .expect("route.open module-rejection refusal event");
11405 let logged = event.fields.get("module_code").expect("module_code field");
11406 assert!(!logged.bytes().any(|byte| byte < 0x20));
11407 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
11408 }
11409
11410 #[tokio::test]
11411 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
11412 let registry = Arc::new(Registry::default());
11413 let supervisor_handle = SupervisorHandle::new();
11414 let missing_program = std::env::temp_dir().join(format!(
11415 "subc-route-open-missing-program-{}",
11416 std::process::id()
11417 ));
11418 let supervisor =
11419 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11420 .with_handle(supervisor_handle.clone());
11421 let module = supervisor
11422 .supervise_configured(
11423 ModuleSpec {
11424 module_id: "failed".to_string(),
11425 program: missing_program,
11426 args: Vec::new(),
11427 env: Vec::new(),
11428 reserved: false,
11429 reserved_prefixes: Vec::new(),
11430 protocol: ModuleProtocol::Subc,
11431 overlap: Default::default(),
11432 },
11433 true,
11434 )
11435 .unwrap();
11436 assert_eq!(module.state().unwrap(), ModuleState::Failed);
11437
11438 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11439 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
11440 let response = handler
11441 .handle_control_frame(
11442 &ctx,
11443 route_open_frame(305, "failed", unique_project_root("failed")),
11444 )
11445 .await
11446 .unwrap();
11447
11448 assert_eq!(response[0].header.ty, FrameType::Error);
11449 let error = parse_error(&response[0]);
11450 assert_eq!(error["code"], "target_unavailable");
11451 assert!(error["message"]
11452 .as_str()
11453 .unwrap()
11454 .contains("state=failed, enabled=true, live=false"));
11455 }
11456
11457 #[tokio::test]
11458 async fn route_open_role_mismatch_remains_target_unavailable() {
11459 let registry = Arc::new(Registry::default());
11460 let handler = ControlHandler::new(Arc::clone(®istry));
11461 handler
11462 .handle_control(
11463 ConnectionId::new(41),
11464 non_routable_hello_frame_with_control_ops("health-only", 306, None),
11465 )
11466 .unwrap();
11467
11468 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
11469 let response = handler
11470 .handle_control_frame(
11471 &ctx,
11472 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
11473 )
11474 .await
11475 .unwrap();
11476
11477 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11478 assert!(parse_error(&response[0])["message"]
11479 .as_str()
11480 .unwrap()
11481 .contains("does not provide the requested target"));
11482 }
11483
11484 #[tokio::test]
11485 async fn route_open_inactive_registration_remains_target_unavailable() {
11486 let registry = Arc::new(Registry::default());
11487 let handler = ControlHandler::new(Arc::clone(®istry));
11488 handler
11489 .handle_control(
11490 ConnectionId::new(43),
11491 hello_frame("inactive", PROTOCOL_VERSION, 308),
11492 )
11493 .unwrap();
11494 assert!(registry
11495 .set_module_state_for_test("inactive", ChannelState::Closed)
11496 .unwrap());
11497
11498 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
11499 let response = handler
11500 .handle_control_frame(
11501 &ctx,
11502 route_open_frame(309, "inactive", unique_project_root("inactive")),
11503 )
11504 .await
11505 .unwrap();
11506
11507 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11508 assert!(parse_error(&response[0])["message"]
11509 .as_str()
11510 .unwrap()
11511 .contains("is not active"));
11512 }
11513
11514 #[tokio::test]
11515 async fn late_health_reply_is_recorded_through_the_module_response_path() {
11516 let registry = Arc::new(Registry::default());
11517 let forwarding = Arc::new(ForwardingTable::default());
11518 let supervisor_handle = SupervisorHandle::new();
11519 let supervisor =
11520 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
11521 .with_forwarding(Arc::clone(&forwarding))
11522 .with_handle(supervisor_handle.clone());
11523 let module = supervisor
11524 .supervise_configured(
11525 crate::ModuleSpec {
11526 module_id: "late-health-response".to_string(),
11527 program: PathBuf::from("disabled-module"),
11528 args: Vec::new(),
11529 env: Vec::new(),
11530 reserved: false,
11531 reserved_prefixes: Vec::new(),
11532 protocol: ModuleProtocol::Subc,
11533 overlap: Default::default(),
11534 },
11535 false,
11536 )
11537 .unwrap();
11538 let handler =
11539 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11540 .with_supervisor(supervisor_handle);
11541 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
11542 handler
11543 .handle_control_frame(
11544 &module_ctx,
11545 hello_frame_with_control_ops(
11546 "late-health-response",
11547 PROTOCOL_VERSION,
11548 7,
11549 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11550 ),
11551 )
11552 .await
11553 .unwrap();
11554 let probe_started_at = Instant::now() - Duration::from_millis(80);
11555 let pending = forwarding
11556 .begin_health_probe_rpc_for(
11557 "late-health-response",
11558 MODULE_CONTROL_OP_HEALTH_CHECK,
11559 probe_started_at,
11560 Instant::now() - Duration::from_millis(1),
11561 )
11562 .unwrap();
11563 assert!(forwarding
11564 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
11565 .unwrap());
11566
11567 let responses = handler
11568 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
11569 .await
11570 .unwrap();
11571
11572 assert!(responses.is_empty());
11573 let health = module.status().unwrap().health;
11574 assert_eq!(health.late_answer_count, 1);
11575 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
11576 }
11577
11578 #[tokio::test]
11579 async fn health_probe_timeout_and_module_death_are_typed() {
11580 let registry = Arc::new(Registry::default());
11581 let forwarding = Arc::new(ForwardingTable::default());
11582 let handler =
11583 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11584 .with_health_probe_timeout(Duration::from_millis(50));
11585 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11586 hello_via_sink(
11587 &handler,
11588 &module_ctx,
11589 &mut module_rx,
11590 hello_frame_with_control_ops(
11591 "aft",
11592 PROTOCOL_VERSION,
11593 7,
11594 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11595 ),
11596 )
11597 .await;
11598
11599 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11600 let responses = handler
11601 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
11602 .await
11603 .unwrap();
11604 assert_eq!(responses[0].header.ty, FrameType::Error);
11605 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
11606 let _ = module_rx.try_recv();
11607
11608 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
11609 let health_handler = handler.clone();
11610 let death_task = tokio::spawn(async move {
11611 health_handler
11612 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
11613 .await
11614 .unwrap()
11615 });
11616 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11617 .await
11618 .unwrap()
11619 .unwrap();
11620 handler
11621 .cleanup_connection(module_ctx.connection_id)
11622 .unwrap();
11623 let responses = death_task.await.unwrap();
11624 assert_eq!(responses[0].header.ty, FrameType::Error);
11625 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
11626 }
11627
11628 #[test]
11629 fn hello_requires_exact_protocol_version() {
11630 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
11631 let registry = Arc::new(Registry::default());
11632 let handler = ControlHandler::new(Arc::clone(®istry));
11633 let responses = handler
11634 .handle_control(
11635 ConnectionId::new(connection),
11636 hello_frame("aft", offered, 9),
11637 )
11638 .unwrap();
11639
11640 assert_eq!(responses.len(), 1);
11641 assert_eq!(responses[0].header.ty, FrameType::Error);
11642 let error = parse_error(&responses[0]);
11643 assert_eq!(error["code"], "version_unsupported");
11644 assert!(registry.get_module("aft").unwrap().is_none());
11645 assert_eq!(registry.active_registration_count().unwrap(), 0);
11646 }
11647 }
11648
11649 #[test]
11650 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
11651 let registry = Arc::new(Registry::default());
11652 let forwarding = Arc::new(ForwardingTable::default());
11653 let handler =
11654 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11655 let module_connection = ConnectionId::new(301);
11656 let registration = registry
11657 .register_with_control_ops(
11658 manifest("aft-push", PROTOCOL_VERSION),
11659 PROTOCOL_VERSION,
11660 module_connection,
11661 module_baseline_control_ops(),
11662 )
11663 .unwrap();
11664 let (module_tx, _module_rx) = mpsc::channel(8);
11665 let endpoint = forwarding
11666 .register_module_connection(
11667 module_connection,
11668 "aft-push".to_string(),
11669 PROTOCOL_VERSION,
11670 manifest_concurrency(®istration.manifest),
11671 FrameSink::new(module_tx),
11672 )
11673 .unwrap();
11674
11675 let unknown = Frame::build(
11677 FrameType::Push,
11678 control_flags(),
11679 0,
11680 0,
11681 5,
11682 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
11683 )
11684 .unwrap();
11685 let out = handler.handle_status_update(endpoint, unknown).unwrap();
11686 assert!(
11687 out.is_empty(),
11688 "unknown push op must be ignored, got {out:?}"
11689 );
11690
11691 let malformed = Frame::build(
11693 FrameType::Push,
11694 control_flags(),
11695 0,
11696 0,
11697 6,
11698 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
11699 )
11700 .unwrap();
11701 let out = handler.handle_status_update(endpoint, malformed).unwrap();
11702 assert_eq!(out.len(), 1);
11703 assert_eq!(out[0].header.ty, FrameType::Error);
11704 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
11705 }
11706
11707 #[test]
11708 fn hello_rejected_when_connection_already_owns_client_routes() {
11709 let registry = Arc::new(Registry::default());
11710 let forwarding = Arc::new(ForwardingTable::default());
11711 let handler =
11712 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11713 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
11715 let client_connection = ConnectionId::new(202);
11716
11717 let responses = handler
11720 .handle_control(
11721 client_connection,
11722 hello_frame("aft-second", PROTOCOL_VERSION, 9),
11723 )
11724 .unwrap();
11725 assert_eq!(responses[0].header.ty, FrameType::Error);
11726 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
11727 assert!(registry.get_module("aft-second").unwrap().is_none());
11728 }
11729
11730 #[tokio::test]
11731 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
11732 let registry = Arc::new(Registry::default());
11733 let forwarding = Arc::new(ForwardingTable::default());
11734 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
11735 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
11736 hello_via_sink(
11737 &handler,
11738 &module_ctx,
11739 &mut module_rx,
11740 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
11741 )
11742 .await;
11743 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
11744 let pending = forwarding
11745 .begin_route_bind_relay_for_test(
11746 client_ctx.connection_id,
11747 client_ctx.egress.clone(),
11748 2,
11749 "alpha",
11750 )
11751 .unwrap();
11752 forwarding
11753 .complete_pending_relay(
11754 module_ctx.connection_id,
11755 pending.corr,
11756 RouteBindRelayOutcome::Accepted,
11757 )
11758 .unwrap();
11759 client_rx.try_recv().unwrap();
11760 for module_id in ["beta", "alpha"] {
11761 let replies = handler
11762 .handle_control_frame(
11763 &module_ctx,
11764 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
11765 )
11766 .await
11767 .unwrap();
11768 assert_eq!(replies.len(), 1, "second HELLO must be refused");
11769 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
11770 }
11771 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
11772 assert!(registry.get_module("beta").unwrap().is_none());
11773 assert!(matches!(
11774 forwarding
11775 .lookup_data_route(
11776 client_ctx.connection_id,
11777 pending.client_channel,
11778 pending.client_epoch,
11779 )
11780 .unwrap(),
11781 DataRoute::Client(DataRouteState::Bound(_))
11782 ));
11783 assert!(handler
11784 .hello_launch_nonces
11785 .lock()
11786 .unwrap()
11787 .presented(module_ctx.connection_id, Some("alpha-nonce")));
11788 assert!(module_rx.try_recv().is_err());
11789 }
11790
11791 #[test]
11792 fn reserved_module_hello_requires_matching_launch_nonce() {
11793 let registry = Arc::new(Registry::default());
11794 let supervisor = SupervisorHandle::new();
11795 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
11798 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11799
11800 let no_nonce = handler
11802 .handle_control(
11803 ConnectionId::new(1),
11804 hello_frame("vault", PROTOCOL_VERSION, 1),
11805 )
11806 .unwrap();
11807 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
11808 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
11809 assert!(registry.get_module("vault").unwrap().is_none());
11810
11811 let wrong = handler
11813 .handle_control(
11814 ConnectionId::new(2),
11815 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
11816 )
11817 .unwrap();
11818 assert_eq!(wrong[0].header.ty, FrameType::Error);
11819 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
11820 assert!(registry.get_module("vault").unwrap().is_none());
11821
11822 let ok = handler
11824 .handle_control(
11825 ConnectionId::new(3),
11826 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
11827 )
11828 .unwrap();
11829 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
11830 assert!(registry.get_module("vault").unwrap().is_some());
11831 }
11832
11833 #[test]
11834 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
11835 let registry = Arc::new(Registry::default());
11836 let supervisor = SupervisorHandle::new();
11837 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11838 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11839 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11840
11841 let squat = handler
11842 .handle_control(
11843 ConnectionId::new(1),
11844 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
11845 )
11846 .unwrap();
11847 assert_eq!(squat[0].header.ty, FrameType::Error);
11848 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
11849 assert!(parse_error(&squat[0])["message"]
11850 .as_str()
11851 .unwrap()
11852 .contains("fed:"));
11853
11854 let accepted_peer = handler
11855 .handle_control(
11856 ConnectionId::new(2),
11857 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
11858 )
11859 .unwrap();
11860 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
11861
11862 let accepted_short = handler
11863 .handle_control(
11864 ConnectionId::new(3),
11865 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
11866 )
11867 .unwrap();
11868 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
11869
11870 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
11871 let response = handler
11872 .handle_control(
11873 ConnectionId::new(conn),
11874 hello_frame(module_id, PROTOCOL_VERSION, conn),
11875 )
11876 .unwrap();
11877 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
11878 }
11879 }
11880
11881 #[test]
11882 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
11883 let registry = Arc::new(Registry::default());
11884 let supervisor = SupervisorHandle::new();
11885 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11886 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11887 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
11888 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11889
11890 let owner_nonce = handler
11891 .handle_control(
11892 ConnectionId::new(1),
11893 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
11894 )
11895 .unwrap();
11896 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
11897 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
11898 assert!(registry.get_module("fed:special").unwrap().is_none());
11899
11900 let exact_nonce = handler
11901 .handle_control(
11902 ConnectionId::new(2),
11903 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
11904 )
11905 .unwrap();
11906 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
11907 assert!(registry.get_module("fed:special").unwrap().is_some());
11908 }
11909
11910 #[test]
11911 fn non_reserved_module_ignores_launch_nonce() {
11912 let registry = Arc::new(Registry::default());
11913 let handler = ControlHandler::new(Arc::clone(®istry));
11916 let no_nonce = handler
11917 .handle_control(
11918 ConnectionId::new(1),
11919 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
11920 )
11921 .unwrap();
11922 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
11923 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
11924
11925 let echoed_nonce = handler
11926 .handle_control(
11927 ConnectionId::new(2),
11928 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
11929 )
11930 .unwrap();
11931 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
11932 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
11933 }
11934
11935 #[test]
11936 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
11937 let handler = ControlHandler::default();
11938 let conn = ConnectionId::new(1);
11939 let malformed = Frame::build(
11940 FrameType::Hello,
11941 control_flags(),
11942 0,
11943 0,
11944 3,
11945 b"{not json".to_vec(),
11946 )
11947 .unwrap();
11948
11949 let error = handler.handle_control(conn, malformed).unwrap();
11950 assert_eq!(error[0].header.ty, FrameType::Error);
11951 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
11952
11953 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11954 let pong = handler.handle_control(conn, ping).unwrap();
11955 assert_eq!(pong[0].header.ty, FrameType::Pong);
11956 assert_eq!(pong[0].header.corr, 4);
11957 }
11958
11959 #[test]
11960 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
11961 let registry = Arc::new(Registry::default());
11962 let handler = ControlHandler::new(Arc::clone(®istry));
11963
11964 handler
11965 .handle_control(
11966 ConnectionId::new(1),
11967 hello_frame("aft", PROTOCOL_VERSION, 1),
11968 )
11969 .unwrap();
11970 let duplicate = handler
11971 .handle_control(
11972 ConnectionId::new(2),
11973 hello_frame("aft", PROTOCOL_VERSION, 2),
11974 )
11975 .unwrap();
11976
11977 assert_eq!(duplicate[0].header.ty, FrameType::Error);
11978 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11979 let registration = registry.get_module("aft").unwrap().unwrap();
11980 assert_eq!(registration.connection_id, ConnectionId::new(1));
11981 }
11982
11983 #[test]
11984 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
11985 let registry = Arc::new(Registry::default());
11986 let forwarding = Arc::new(ForwardingTable::default());
11987 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
11988 let handler =
11989 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11990 .with_process_liveness(process_liveness);
11991 let (ctx, route_channel, route_epoch) =
11992 bind_liveness_route(®istry, &forwarding, "aft-dead");
11993 let responses = handler
11994 .handle_route_poll(
11995 &ctx,
11996 route_poll_frame(41, PollKind::Liveness, route_channel),
11997 route_channel,
11998 route_epoch,
11999 PollKind::Liveness,
12000 )
12001 .unwrap();
12002
12003 assert_eq!(responses.len(), 1);
12004 assert_eq!(responses[0].header.ty, FrameType::Response);
12005 assert_route_poll_liveness(&responses[0], false);
12006 }
12007
12008 #[test]
12009 fn liveness_poll_without_process_source_uses_bound_route() {
12010 let registry = Arc::new(Registry::default());
12011 let forwarding = Arc::new(ForwardingTable::default());
12012 let handler =
12013 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12014 let (ctx, route_channel, route_epoch) =
12015 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
12016 let responses = handler
12017 .handle_route_poll(
12018 &ctx,
12019 route_poll_frame(42, PollKind::Liveness, route_channel),
12020 route_channel,
12021 route_epoch,
12022 PollKind::Liveness,
12023 )
12024 .unwrap();
12025
12026 assert_route_poll_liveness(&responses[0], true);
12027 }
12028
12029 #[test]
12030 fn liveness_poll_untracked_process_source_uses_bound_route() {
12031 let registry = Arc::new(Registry::default());
12032 let forwarding = Arc::new(ForwardingTable::default());
12033 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
12034 let handler =
12035 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12036 .with_process_liveness(process_liveness);
12037 let (ctx, route_channel, route_epoch) =
12038 bind_liveness_route(®istry, &forwarding, "aft-untracked");
12039 let responses = handler
12040 .handle_route_poll(
12041 &ctx,
12042 route_poll_frame(43, PollKind::Liveness, route_channel),
12043 route_channel,
12044 route_epoch,
12045 PollKind::Liveness,
12046 )
12047 .unwrap();
12048
12049 assert_route_poll_liveness(&responses[0], true);
12050 }
12051
12052 #[tokio::test]
12053 async fn unknown_op_returns_unknown_control_op() {
12054 let handler = ControlHandler::default();
12055 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
12056 let request = Frame::build(
12057 FrameType::Request,
12058 control_flags(),
12059 0,
12060 0,
12061 55,
12062 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
12063 )
12064 .unwrap();
12065
12066 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12067
12068 assert_eq!(response.len(), 1);
12069 assert_eq!(response[0].header.ty, FrameType::Error);
12070 assert_eq!(response[0].header.corr, 55);
12071 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
12072 }
12073
12074 #[tokio::test]
12075 async fn supervisor_provenance_rejects_unknown_exact_module() {
12076 let handler = ControlHandler::default();
12077 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
12078 let request = Frame::build(
12079 FrameType::Request,
12080 control_flags(),
12081 0,
12082 0,
12083 57,
12084 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
12085 )
12086 .unwrap();
12087
12088 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12089
12090 assert_eq!(response.len(), 1);
12091 assert_eq!(response[0].header.ty, FrameType::Error);
12092 assert_eq!(response[0].header.corr, 57);
12093 let error = parse_error(&response[0]);
12094 assert_eq!(error["code"], "unknown_module");
12095 assert_eq!(error["message"], "module_id 'missing' is not supervised");
12096 }
12097
12098 #[test]
12099 fn provenance_probe_override_keeps_handler_tests_deterministic() {
12100 let expected = subc_control::RunningImageAgreement::Unavailable {
12101 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12102 };
12103 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
12104 assert_eq!(handler.provenance_probe_override, Some(expected));
12105 }
12106
12107 #[test]
12108 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
12109 let verdict = reload_verdict(
12110 std::path::Path::new("/bin/new"),
12111 Some(std::path::Path::new("/bin/old")),
12112 subc_control::RunningImageAgreement::Unavailable {
12113 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12114 },
12115 );
12116 assert!(matches!(
12117 verdict.path,
12118 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
12119 if configured == std::path::Path::new("/bin/new")
12120 && spawned_from == std::path::Path::new("/bin/old")
12121 ));
12122 }
12123
12124 #[cfg(windows)]
12125 #[test]
12126 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
12127 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
12128 let configured = dir.join("module.exe");
12129 std::fs::write(&configured, b"executable path fixture").unwrap();
12130 let spawned = std::fs::canonicalize(&configured).unwrap();
12131 assert_ne!(
12132 configured, spawned,
12133 "fixture must exercise different path spellings"
12134 );
12135 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
12136 let lower_case = configured.to_string_lossy().to_lowercase();
12137 for spelling in [
12138 &configured,
12139 Path::new(&forward_slashes),
12140 Path::new(&lower_case),
12141 ] {
12142 let verdict = reload_verdict(
12143 spelling,
12144 Some(&spawned),
12145 subc_control::RunningImageAgreement::Unavailable {
12146 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12147 },
12148 );
12149 assert_eq!(
12150 verdict.path,
12151 ReloadPathAgreement::Match,
12152 "configured {spelling:?}, spawned {spawned:?}"
12153 );
12154 }
12155 }
12156
12157 #[cfg(windows)]
12158 #[test]
12159 fn windows_reload_path_keeps_different_existing_program_pending() {
12160 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
12161 let old = dir.join("old.exe");
12162 let new = dir.join("new.exe");
12163 std::fs::write(&old, b"same executable content").unwrap();
12164 std::fs::copy(&old, &new).unwrap();
12165 let spawned = std::fs::canonicalize(&old).unwrap();
12166 let image = subc_control::RunningImageAgreement::Unavailable {
12167 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12168 };
12169 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
12170 assert!(
12171 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
12172 if configured == new && spawned_from == spawned)
12173 );
12174 let missing = dir.join("missing.exe");
12175 assert!(matches!(
12176 reload_verdict(&missing, Some(&spawned), image).path,
12177 ReloadPathAgreement::Mismatch { .. }
12178 ));
12179 }
12180
12181 #[test]
12182 fn reload_verdict_detects_replaced_image_at_same_path() {
12183 let image = subc_control::RunningImageAgreement::Mismatch {
12184 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
12185 digest: "old".into(),
12186 },
12187 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
12188 digest: "new".into(),
12189 },
12190 };
12191 let verdict = reload_verdict(
12192 std::path::Path::new("/bin/same"),
12193 Some(std::path::Path::new("/bin/same")),
12194 image.clone(),
12195 );
12196 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12197 assert_eq!(verdict.image, image);
12198 }
12199
12200 #[test]
12201 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
12202 let image = subc_control::RunningImageAgreement::Unavailable {
12203 reason: subc_control::RunningImageUnavailableReason::NotRunning,
12204 };
12205 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
12206 assert_eq!(
12207 verdict.path,
12208 subc_control::ReloadPathAgreement::Unavailable {
12209 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
12210 }
12211 );
12212 assert_eq!(verdict.image, image);
12213
12214 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
12215 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12216 };
12217 let verdict = reload_verdict(
12218 std::path::Path::new("/bin/same"),
12219 Some(std::path::Path::new("/bin/same")),
12220 unconfirmed.clone(),
12221 );
12222 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12223 assert_eq!(verdict.image, unconfirmed);
12224 }
12225
12226 #[test]
12227 fn reload_verdict_preserves_each_image_unavailability_reason() {
12228 use subc_control::RunningImageUnavailableReason as Reason;
12229
12230 for reason in [
12231 Reason::NotRunning,
12232 Reason::UnsupportedPlatform,
12233 Reason::RunningExecutableUnreadable,
12234 Reason::SpawnedPathUnreadable,
12235 Reason::HashFailed,
12236 Reason::ProcessIdentityUnconfirmed,
12237 Reason::Unknown("future_probe_reason".to_string()),
12238 ] {
12239 let image = subc_control::RunningImageAgreement::Unavailable {
12240 reason: reason.clone(),
12241 };
12242 let verdict = reload_verdict(
12243 std::path::Path::new("/bin/same"),
12244 Some(std::path::Path::new("/bin/same")),
12245 image.clone(),
12246 );
12247 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12248 assert_eq!(verdict.image, image, "{reason:?}");
12249 }
12250 }
12251
12252 #[tokio::test]
12253 async fn malformed_control_bodies_return_invalid_control_body() {
12254 let handler = ControlHandler::default();
12255 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
12256
12257 for (corr, body) in [
12258 (56, br#"{"route_channel":1}"#.as_slice()),
12259 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
12260 (
12261 58,
12262 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
12263 ),
12264 ] {
12265 let request = Frame::build(
12266 FrameType::Request,
12267 control_flags(),
12268 0,
12269 0,
12270 corr,
12271 body.to_vec(),
12272 )
12273 .unwrap();
12274 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12275
12276 assert_eq!(response.len(), 1);
12277 assert_eq!(response[0].header.ty, FrameType::Error);
12278 assert_eq!(response[0].header.corr, corr);
12279 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
12280 }
12281 }
12282
12283 #[tokio::test]
12284 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
12285 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12286 let registry = Arc::new(Registry::default());
12287 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
12288 let router = Router::with_control_handler(Arc::clone(&control));
12289 let connection = router.begin_connection();
12290 let (ctx, mut rx) = route_ctx(connection.id());
12291
12292 router
12293 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
12294 .await
12295 .unwrap();
12296 let response = rx.recv().await.unwrap();
12297 let ack = parse_ack(&response);
12298 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
12299 let channel = 1;
12300
12301 let goodbye =
12302 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
12303 router.route_for_connection(&ctx, goodbye).await.unwrap();
12304 assert!(rx.try_recv().is_err());
12305 assert!(registry.get_module("aft").unwrap().is_none());
12306
12307 router
12308 .route_for_connection(&ctx, channel_request(channel, 13))
12309 .await
12310 .unwrap();
12311 let error_frame = rx.recv().await.unwrap();
12312 assert_eq!(error_frame.header.ty, FrameType::Error);
12313 assert_eq!(error_frame.header.channel, channel);
12314 let captured = crate::router::test_log::captured_logs(&logs);
12315 assert_eq!(
12316 captured
12317 .lines()
12318 .filter(|line| {
12319 line.contains("module_id=aft")
12320 && line.contains("reason=explicit_goodbye")
12321 && line.contains("module registration ended")
12322 })
12323 .count(),
12324 1,
12325 "unexpected GOODBYE registry log: {captured}"
12326 );
12327 }
12328
12329 #[tokio::test]
12330 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
12331 let registry = Arc::new(Registry::default());
12332 let handler = ControlHandler::new(registry).with_capability_config(
12333 [("prov".to_string(), true), ("cons".to_string(), true)],
12334 BTreeMap::new(),
12335 );
12336 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
12337 register_capability_manifest(
12338 &handler,
12339 &provider_ctx,
12340 &mut provider_rx,
12341 capability_manifest("prov", &["thing/v1"], &[]),
12342 1,
12343 )
12344 .await;
12345 let mut consumer = capability_manifest("cons", &[], &[]);
12346 consumer.capabilities.as_mut().unwrap().requires.push(
12347 subc_protocol::manifest::CapabilityRequirement {
12348 capability: "thing/v1".to_string(),
12349 need: subc_protocol::manifest::CapabilityNeed::Required,
12350 },
12351 );
12352 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
12353 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
12354 assert_eq!(
12355 handler.capability_evaluator.verdict("cons", "thing/v1"),
12356 Some(CapabilityVerdict::Provided)
12357 );
12358 let (mut client_rx, _) = open_route_for_capability_test(
12359 &handler,
12360 &provider_ctx,
12361 &mut provider_rx,
12362 703,
12363 3,
12364 "prov",
12365 None,
12366 )
12367 .await;
12368 let goodbye =
12369 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12370 handler
12371 .handle_control_frame(&provider_ctx, goodbye)
12372 .await
12373 .unwrap();
12374 assert_eq!(
12375 handler.capability_evaluator.verdict("cons", "thing/v1"),
12376 Some(CapabilityVerdict::NeverProvided)
12377 );
12378 let closed = client_rx
12379 .try_recv()
12380 .expect("GOODBYE pushes route.closed before route GOODBYE");
12381 assert!(
12382 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
12383 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
12384 );
12385 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
12386 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
12387 }
12388
12389 struct RouteClosedFixture {
12390 handler: ControlHandler,
12391 #[cfg(unix)]
12392 supervisor: Supervisor,
12393 module_ctx: RouteCtx,
12394 #[cfg(unix)]
12395 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12396 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12397 route: Arc<crate::forwarding::RouteBinding>,
12398 }
12399
12400 async fn route_closed_fixture() -> RouteClosedFixture {
12401 let registry = Arc::new(Registry::default());
12402 let handler = ControlHandler::new(Arc::clone(®istry));
12403 #[cfg(unix)]
12404 let supervisor = Supervisor::new(registry, RestartPolicy::default())
12405 .with_forwarding(Arc::clone(&handler.forwarding));
12406 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
12407 register_capability_manifest(
12408 &handler,
12409 &module_ctx,
12410 &mut module_rx,
12411 capability_manifest("prov", &[], &[]),
12412 1,
12413 )
12414 .await;
12415 let (client_rx, bind) = open_route_for_capability_test(
12416 &handler,
12417 &module_ctx,
12418 &mut module_rx,
12419 702,
12420 2,
12421 "prov",
12422 None,
12423 )
12424 .await;
12425 let ModuleControlRequest::RouteBind {
12426 route_channel: channel,
12427 epoch: route_epoch,
12428 ..
12429 } = bind
12430 else {
12431 panic!("expected route.bind");
12432 };
12433 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
12434 handler
12435 .forwarding
12436 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
12437 .unwrap()
12438 else {
12439 panic!("the acknowledged route must be bound");
12440 };
12441 RouteClosedFixture {
12442 handler,
12443 #[cfg(unix)]
12444 supervisor,
12445 module_ctx,
12446 #[cfg(unix)]
12447 module_rx,
12448 client_rx,
12449 route,
12450 }
12451 }
12452
12453 impl RouteClosedFixture {
12454 #[cfg(unix)]
12455 async fn cut(&mut self) {
12456 let notices = async {
12457 let mut draining = self.module_rx.recv().await.unwrap();
12458 assert!(matches!(
12459 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
12460 &draining.body
12461 )
12462 .unwrap(),
12463 subc_protocol::session::ModuleControlCommand::Draining {
12464 reason: RouteCloseReason::Restart,
12465 ..
12466 }
12467 ));
12468 draining.flushed.take().unwrap().send(()).unwrap();
12469 let mut closing = self.client_rx.recv().await.unwrap();
12470 assert!(matches!(
12471 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
12472 ClientControlPush::RouteClosing {
12473 reason: RouteCloseReason::Restart,
12474 ..
12475 }
12476 ));
12477 closing.flushed.take().unwrap().send(()).unwrap();
12478 };
12479 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
12480 result.unwrap();
12481 }
12482
12483 fn disconnect(&mut self) -> ClientControlPush {
12484 self.handler
12485 .cleanup_connection(self.module_ctx.connection_id)
12486 .unwrap();
12487 let closed = self.client_rx.try_recv().unwrap();
12488 assert_eq!(closed.header.ty, FrameType::Push);
12489 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
12490 assert!(matches!(
12491 &push,
12492 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
12493 if channels == &[self.route.client_channel]
12494 ));
12495 assert_eq!(
12496 self.client_rx.try_recv().unwrap().header.ty,
12497 FrameType::Goodbye
12498 );
12499 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
12500 push
12501 }
12502 }
12503
12504 #[cfg(unix)]
12505 #[tokio::test(start_paused = true)]
12506 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
12507 let mut fixture = route_closed_fixture().await;
12508 fixture.cut().await;
12509 assert!(matches!(
12510 fixture.disconnect(),
12511 ClientControlPush::RouteClosed {
12512 reason: RouteCloseReason::Restart,
12513 drained: true,
12514 ..
12515 }
12516 ));
12517 }
12518
12519 #[cfg(unix)]
12520 #[tokio::test(start_paused = true)]
12521 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
12522 let mut fixture = route_closed_fixture().await;
12523 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
12524 let started = tokio::time::Instant::now();
12525 fixture.cut().await;
12526 assert!(started.elapsed() >= Duration::from_secs(2));
12527 assert_eq!(fixture.route.flow.in_flight(), 1);
12528 assert!(matches!(
12529 fixture.disconnect(),
12530 ClientControlPush::RouteClosed {
12531 reason: RouteCloseReason::Restart,
12532 drained: false,
12533 ..
12534 }
12535 ));
12536 }
12537
12538 #[tokio::test]
12539 async fn module_connection_crash_reports_crash_and_not_drained() {
12540 let mut fixture = route_closed_fixture().await;
12541 assert!(matches!(
12545 fixture.disconnect(),
12546 ClientControlPush::RouteClosed {
12547 reason: RouteCloseReason::Crash,
12548 drained: false,
12549 ..
12550 }
12551 ));
12552 }
12553
12554 #[tokio::test]
12555 async fn dropping_router_connection_releases_registration() {
12556 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12557 let registry = Arc::new(Registry::default());
12558 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
12559 let router = Router::with_control_handler(Arc::clone(&control));
12560 let connection = router.begin_connection();
12561 let connection_id = connection.id();
12562 let (ctx, mut rx) = route_ctx(connection_id);
12563
12564 router
12565 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
12566 .await
12567 .unwrap();
12568 let response = rx.recv().await.unwrap();
12569 let ack = parse_ack(&response);
12570 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
12571 assert!(registry.get_module("aft").unwrap().is_some());
12572
12573 drop(connection);
12574
12575 assert!(registry.get_module("aft").unwrap().is_none());
12576 assert_eq!(registry.active_registration_count().unwrap(), 0);
12577
12578 control
12579 .cleanup_connection(ConnectionId::new(u64::MAX))
12580 .unwrap();
12581 let captured = crate::router::test_log::captured_logs(&logs);
12582 println!("captured registry lifecycle logs:\n{captured}");
12583 let events: Vec<_> = captured
12584 .lines()
12585 .filter(|line| {
12586 line.contains("module registered module_id=aft ")
12587 || line.contains("module registration ended")
12588 })
12589 .collect();
12590 assert_eq!(
12591 events.len(),
12592 2,
12593 "unexpected registry lifecycle logs: {captured}"
12594 );
12595 assert!(events[0].contains("module registered module_id=aft "));
12596 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
12597 assert!(events[1].contains(&format!(
12598 "module_id=aft connection_id={}",
12599 connection_id.get()
12600 )));
12601 assert!(events[1].contains("reason=connection_closed"));
12602 assert!(events[1].contains("module registration ended"));
12603 assert_eq!(
12604 captured
12605 .lines()
12606 .filter(|line| line.contains("module registered module_id=aft "))
12607 .count(),
12608 1,
12609 "legacy registration admission line must appear once: {captured}"
12610 );
12611 }
12612
12613 #[tokio::test]
12614 async fn hello_registration_keeps_legacy_module_registered_line_once() {
12615 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12616 let registry = Arc::new(Registry::default());
12617 let control = ControlHandler::new(Arc::clone(®istry));
12618 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
12619 hello_via_sink(
12620 &control,
12621 &ctx,
12622 &mut rx,
12623 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
12624 )
12625 .await;
12626
12627 let captured = crate::router::test_log::captured_logs(&logs);
12628 let admissions: Vec<_> = captured
12629 .lines()
12630 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
12631 .collect();
12632 assert_eq!(
12633 admissions.len(),
12634 1,
12635 "legacy admission line must remain exactly once: {captured}"
12636 );
12637 assert!(admissions[0].contains("routable_provider=true"));
12638 assert!(admissions[0].contains("connection_id=777"));
12639 }
12640
12641 fn capability_manifest(
12642 module_id: &str,
12643 provides: &[&str],
12644 must_never_reach: &[&str],
12645 ) -> ModuleManifest {
12646 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
12647 manifest.capabilities = Some(CapabilityDeclarations {
12648 provides: provides
12649 .iter()
12650 .map(|capability| (*capability).to_string())
12651 .collect(),
12652 requires: Vec::new(),
12653 must_never_reach: must_never_reach
12654 .iter()
12655 .map(|capability| (*capability).to_string())
12656 .collect(),
12657 });
12658 manifest
12659 }
12660
12661 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
12662 Frame::build(
12663 FrameType::Hello,
12664 control_flags(),
12665 0,
12666 0,
12667 corr,
12668 serde_json::to_vec(&ModuleHelloBody {
12669 protocol_ver: manifest.protocol_ver,
12670 manifest,
12671 control_ops: None,
12672 launch_nonce: None,
12673 })
12674 .expect("capability test HELLO serializes"),
12675 )
12676 .expect("capability test HELLO frame builds")
12677 }
12678
12679 fn catalog_update_with_capabilities_frame(
12680 corr: u64,
12681 capabilities: CapabilityDeclarations,
12682 ) -> Frame {
12683 Frame::build(
12684 FrameType::Request,
12685 control_flags(),
12686 0,
12687 0,
12688 corr,
12689 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
12690 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
12691 capabilities: Some(capabilities),
12692 ready: None,
12693 })
12694 .expect("capability catalog.update serializes"),
12695 )
12696 .expect("capability catalog.update frame builds")
12697 }
12698
12699 async fn register_capability_manifest(
12700 handler: &ControlHandler,
12701 ctx: &RouteCtx,
12702 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
12703 manifest: ModuleManifest,
12704 corr: u64,
12705 ) {
12706 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
12707 }
12708
12709 async fn open_route_for_capability_test(
12710 handler: &ControlHandler,
12711 target_ctx: &RouteCtx,
12712 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
12713 client_connection_id: u64,
12714 corr: u64,
12715 target_module_id: &str,
12716 consumer_identity: Option<ConsumerIdentity>,
12717 ) -> (
12718 mpsc::Receiver<crate::router::OutboundFrame>,
12719 ModuleControlRequest,
12720 ) {
12721 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
12722 let route_handler = handler.clone();
12723 let target_module_id = target_module_id.to_string();
12724 let route_task = tokio::spawn(async move {
12725 route_handler
12726 .handle_control_frame(
12727 &client_ctx,
12728 route_open_frame_with_admission_facts(
12729 corr,
12730 &target_module_id,
12731 unique_project_root("admission-facts"),
12732 consumer_identity,
12733 None,
12734 ),
12735 )
12736 .await
12737 .expect("capability test route.open succeeds")
12738 });
12739 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
12740 .await
12741 .expect("capability test route.open must reach route.bind")
12742 .expect("target control receiver stays open");
12743 let bind_request: ModuleControlRequest =
12744 serde_json::from_slice(&bind.body).expect("route.bind decodes");
12745 handler
12746 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
12747 .await
12748 .expect("capability test route.bind ACK succeeds");
12749 assert!(route_task.await.expect("route.open task joins").is_empty());
12750 let opened = client_rx
12751 .recv()
12752 .await
12753 .expect("successful route.open publishes a response");
12754 assert!(matches!(
12755 serde_json::from_slice::<ClientControlResponse>(&opened.body),
12756 Ok(ClientControlResponse::RouteOpen { .. })
12757 ));
12758 (client_rx, bind_request)
12759 }
12760
12761 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
12762 assert_eq!(frame.header.ty, FrameType::Push);
12763 assert_eq!(frame.header.channel, 0);
12764 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
12765 .expect("route.closed control push decodes");
12766 let ClientControlPush::RouteClosed { channels, .. } = &push else {
12767 panic!("expected route.closed");
12768 };
12769 assert_eq!(channels.len(), 1, "exactly one violating route closed");
12770 let channels = channels.clone();
12771 assert_eq!(
12772 push,
12773 ClientControlPush::RouteClosed {
12774 module_id: target_module_id.to_string(),
12775 channels,
12776 reason: RouteCloseReason::CapabilityDenied,
12777 drained: false,
12778 abandoned: 0,
12779 excluded_subscriptions: 0,
12780 terminal: Some(false),
12781 }
12782 );
12783 }
12784
12785 #[tokio::test]
12786 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
12787 let registry = Arc::new(Registry::default());
12788 let forwarding = Arc::new(ForwardingTable::default());
12789 let supervisor = SupervisorHandle::new();
12790 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12791 let handler =
12792 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12793 .with_supervisor(supervisor);
12794 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
12795 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
12796 register_capability_manifest(
12797 &handler,
12798 &target_ctx,
12799 &mut target_rx,
12800 capability_manifest("target", &["credentials-provider/v1"], &[]),
12801 1,
12802 )
12803 .await;
12804 register_capability_manifest(
12805 &handler,
12806 &opener_ctx,
12807 &mut opener_rx,
12808 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12809 2,
12810 )
12811 .await;
12812
12813 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
12814 let replies = handler
12815 .handle_control_frame(
12816 &client_ctx,
12817 route_open_frame_with_admission_facts(
12818 3,
12819 "target",
12820 unique_project_root("admission-facts"),
12821 Some(ConsumerIdentity {
12822 module_id: "opener".to_string(),
12823 launch_nonce: "opener-nonce".to_string(),
12824 }),
12825 None,
12826 ),
12827 )
12828 .await
12829 .expect("denied route.open returns a typed frame");
12830 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
12831 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12832 assert!(
12833 target_rx.try_recv().is_err(),
12834 "forbidden route.open must not relay route.bind"
12835 );
12836 }
12837
12838 #[tokio::test]
12839 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
12840 let registry = Arc::new(Registry::default());
12841 let forwarding = Arc::new(ForwardingTable::default());
12842 let supervisor = SupervisorHandle::new();
12843 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12844 let handler =
12845 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12846 .with_supervisor(supervisor);
12847 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
12848 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
12849 register_capability_manifest(
12850 &handler,
12851 &target_ctx,
12852 &mut target_rx,
12853 capability_manifest("target", &["credentials-provider/v1"], &[]),
12854 1,
12855 )
12856 .await;
12857 register_capability_manifest(
12858 &handler,
12859 &old_opener_ctx,
12860 &mut old_opener_rx,
12861 capability_manifest("opener", &[], &[]),
12862 2,
12863 )
12864 .await;
12865 let (mut client_rx, _) = open_route_for_capability_test(
12866 &handler,
12867 &target_ctx,
12868 &mut target_rx,
12869 712,
12870 3,
12871 "target",
12872 Some(ConsumerIdentity {
12873 module_id: "opener".to_string(),
12874 launch_nonce: "opener-nonce".to_string(),
12875 }),
12876 )
12877 .await;
12878 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12879
12880 handler
12881 .cleanup_connection(old_opener_ctx.connection_id)
12882 .expect("old opener registration cleans up");
12883 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
12884 register_capability_manifest(
12885 &handler,
12886 &new_opener_ctx,
12887 &mut new_opener_rx,
12888 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12889 4,
12890 )
12891 .await;
12892
12893 assert_capability_denied_push(
12894 client_rx
12895 .try_recv()
12896 .expect("HELLO deny addition must emit route.closed")
12897 .frame,
12898 "target",
12899 );
12900 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12901 assert!(matches!(
12902 target_rx.try_recv(),
12903 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
12904 ));
12905 }
12906
12907 #[tokio::test]
12908 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
12909 let registry = Arc::new(Registry::default());
12910 let forwarding = Arc::new(ForwardingTable::default());
12911 let supervisor = SupervisorHandle::new();
12912 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12913 let handler =
12914 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12915 .with_supervisor(supervisor);
12916 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
12917 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
12918 register_capability_manifest(
12919 &handler,
12920 &target_ctx,
12921 &mut target_rx,
12922 capability_manifest("target", &[], &[]),
12923 1,
12924 )
12925 .await;
12926 register_capability_manifest(
12927 &handler,
12928 &opener_ctx,
12929 &mut opener_rx,
12930 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12931 2,
12932 )
12933 .await;
12934 let (mut client_rx, _) = open_route_for_capability_test(
12935 &handler,
12936 &target_ctx,
12937 &mut target_rx,
12938 722,
12939 3,
12940 "target",
12941 Some(ConsumerIdentity {
12942 module_id: "opener".to_string(),
12943 launch_nonce: "opener-nonce".to_string(),
12944 }),
12945 )
12946 .await;
12947 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12948
12949 let replies = handler
12950 .handle_control_frame(
12951 &target_ctx,
12952 catalog_update_with_capabilities_frame(
12953 4,
12954 CapabilityDeclarations {
12955 provides: vec!["credentials-provider/v1".to_string()],
12956 requires: Vec::new(),
12957 must_never_reach: Vec::new(),
12958 },
12959 ),
12960 )
12961 .await
12962 .expect("claim catalog.update succeeds");
12963 assert!(matches!(
12964 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
12965 Ok(ModuleControlResponseToModule::CatalogUpdate {})
12966 ));
12967 assert_capability_denied_push(
12968 client_rx
12969 .try_recv()
12970 .expect("claim addition must emit route.closed")
12971 .frame,
12972 "target",
12973 );
12974 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12975 assert!(matches!(
12976 target_rx.try_recv(),
12977 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
12978 ));
12979 }
12980
12981 #[tokio::test]
12982 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
12983 let registry = Arc::new(Registry::default());
12984 let forwarding = Arc::new(ForwardingTable::default());
12985 let supervisor = SupervisorHandle::new();
12986 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12987 let handler =
12988 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12989 .with_supervisor(supervisor);
12990 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
12991 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
12992 register_capability_manifest(
12993 &handler,
12994 &target_ctx,
12995 &mut target_rx,
12996 capability_manifest("target", &["credentials-provider/v1"], &[]),
12997 1,
12998 )
12999 .await;
13000 register_capability_manifest(
13001 &handler,
13002 &opener_ctx,
13003 &mut opener_rx,
13004 capability_manifest("opener", &[], &[]),
13005 2,
13006 )
13007 .await;
13008 let (mut client_rx, _) = open_route_for_capability_test(
13009 &handler,
13010 &target_ctx,
13011 &mut target_rx,
13012 732,
13013 3,
13014 "target",
13015 Some(ConsumerIdentity {
13016 module_id: "opener".to_string(),
13017 launch_nonce: "opener-nonce".to_string(),
13018 }),
13019 )
13020 .await;
13021 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13022
13023 handler
13024 .handle_control_frame(
13025 &target_ctx,
13026 catalog_update_with_capabilities_frame(
13027 4,
13028 CapabilityDeclarations {
13029 provides: Vec::new(),
13030 requires: Vec::new(),
13031 must_never_reach: Vec::new(),
13032 },
13033 ),
13034 )
13035 .await
13036 .expect("claim removal catalog.update succeeds");
13037 assert_eq!(
13038 forwarding.active_binding_count().unwrap(),
13039 1,
13040 "removing an attested target claim must leave the route census unchanged"
13041 );
13042 assert!(
13043 client_rx.try_recv().is_err(),
13044 "claim removal must not emit route.closed capability_denied"
13045 );
13046 assert!(
13047 target_rx.try_recv().is_err(),
13048 "claim removal must not send the target a route GOODBYE"
13049 );
13050 }
13051
13052 #[tokio::test]
13055 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
13056 let registry = Arc::new(Registry::default());
13057 let forwarding = Arc::new(ForwardingTable::default());
13058 let supervisor = SupervisorHandle::new();
13059 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13060 let handler =
13061 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13062 .with_supervisor(supervisor);
13063 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
13064 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
13065 register_capability_manifest(
13066 &handler,
13067 &target_ctx,
13068 &mut target_rx,
13069 capability_manifest("target", &["credentials-provider/v1"], &[]),
13070 1,
13071 )
13072 .await;
13073 register_capability_manifest(
13074 &handler,
13075 &opener_ctx,
13076 &mut opener_rx,
13077 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13078 2,
13079 )
13080 .await;
13081
13082 let (_client_rx, bind) = open_route_for_capability_test(
13083 &handler,
13084 &target_ctx,
13085 &mut target_rx,
13086 742,
13087 3,
13088 "target",
13089 None,
13090 )
13091 .await;
13092 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
13093 panic!("direct scope-honesty route must bind");
13094 };
13095 assert_eq!(principal, Some(Principal::Direct));
13096 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13097 }
13098
13099 #[tokio::test]
13102 async fn must_never_reach_self_route_is_capability_forbidden() {
13103 let registry = Arc::new(Registry::default());
13104 let forwarding = Arc::new(ForwardingTable::default());
13105 let supervisor = SupervisorHandle::new();
13106 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
13107 let handler =
13108 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13109 .with_supervisor(supervisor);
13110 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
13111 register_capability_manifest(
13112 &handler,
13113 &self_ctx,
13114 &mut self_rx,
13115 capability_manifest(
13116 "self-provider",
13117 &["credentials-provider/v1"],
13118 &["credentials-provider/v1"],
13119 ),
13120 1,
13121 )
13122 .await;
13123
13124 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
13125 let replies = handler
13126 .handle_control_frame(
13127 &client_ctx,
13128 route_open_frame_with_admission_facts(
13129 2,
13130 "self-provider",
13131 unique_project_root("admission-facts"),
13132 Some(ConsumerIdentity {
13133 module_id: "self-provider".to_string(),
13134 launch_nonce: "self-nonce".to_string(),
13135 }),
13136 None,
13137 ),
13138 )
13139 .await
13140 .expect("self-route refusal returns a typed frame");
13141 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13142 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13143 assert!(
13144 self_rx.try_recv().is_err(),
13145 "self denial must not relay route.bind"
13146 );
13147 }
13148
13149 #[test]
13150 fn unsupported_channel_zero_frame_returns_error() {
13151 let handler = ControlHandler::default();
13152 let request = Frame::build(
13153 FrameType::Request,
13154 control_flags(),
13155 0,
13156 0,
13157 21,
13158 b"opaque".to_vec(),
13159 )
13160 .unwrap();
13161
13162 let response = handler
13163 .handle_control(ConnectionId::new(1), request)
13164 .unwrap();
13165
13166 assert_eq!(response[0].header.ty, FrameType::Error);
13167 assert_eq!(
13168 parse_error(&response[0])["code"],
13169 "unsupported_control_frame"
13170 );
13171 }
13172
13173 mod swap {
13178 use super::*;
13179
13180 const INCUMBENT: ConnectionId = ConnectionId::new(30);
13181 const CANDIDATE: ConnectionId = ConnectionId::new(40);
13182
13183 struct Swap {
13184 registry: Arc<Registry>,
13185 forwarding: Arc<ForwardingTable>,
13186 handler: ControlHandler,
13187 incumbent_ctx: RouteCtx,
13188 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13189 candidate_ctx: RouteCtx,
13190 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13191 }
13192
13193 async fn swap_with_incumbent() -> Swap {
13194 let registry = Arc::new(Registry::default());
13195 let forwarding = Arc::new(ForwardingTable::default());
13196 let handler =
13197 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13198 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
13199 hello_via_sink(
13200 &handler,
13201 &incumbent_ctx,
13202 &mut incumbent_rx,
13203 hello_frame("aft", PROTOCOL_VERSION, 7),
13204 )
13205 .await;
13206 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
13207 Swap {
13208 registry,
13209 forwarding,
13210 handler,
13211 incumbent_ctx,
13212 incumbent_rx,
13213 candidate_ctx,
13214 candidate_rx,
13215 }
13216 }
13217
13218 fn register_candidate(swap: &Swap, ready: Option<bool>) {
13219 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
13220 candidate_manifest.ready = ready;
13221 let registration = swap
13222 .registry
13223 .register_candidate_with_control_ops(
13224 candidate_manifest,
13225 PROTOCOL_VERSION,
13226 CANDIDATE,
13227 module_baseline_control_ops(),
13228 )
13229 .unwrap();
13230 swap.forwarding
13231 .register_candidate_module_connection(
13232 CANDIDATE,
13233 "aft".to_string(),
13234 PROTOCOL_VERSION,
13235 manifest_concurrency(®istration.manifest),
13236 swap.candidate_ctx.egress.clone(),
13237 )
13238 .unwrap();
13239 }
13240
13241 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
13242 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
13243 swap.registry.promote_candidate("aft").unwrap().unwrap();
13244 cutover.incumbent.unwrap()
13245 }
13246
13247 fn keyed_total(counters: &Value, key: &str) -> u64 {
13248 counters[key]
13249 .as_object()
13250 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
13251 .unwrap_or(0)
13252 }
13253
13254 #[tokio::test]
13255 async fn replacement_logs_old_end_and_new_admission_once() {
13256 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13257 let mut swap = swap_with_incumbent().await;
13258 swap.handler
13259 .supervisor
13260 .open_swap("aft", "candidate-nonce".to_string());
13261 hello_via_sink(
13262 &swap.handler,
13263 &swap.candidate_ctx,
13264 &mut swap.candidate_rx,
13265 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
13266 )
13267 .await;
13268 cutover(&swap);
13269 swap.handler.cleanup_connection(INCUMBENT).unwrap();
13270
13271 let captured = crate::router::test_log::captured_logs(&logs);
13272 println!("captured replacement registry logs:\n{captured}");
13273 let new_admissions: Vec<_> = captured
13274 .lines()
13275 .filter(|line| {
13276 line.contains("connection_id=40")
13277 && line.contains("swap candidate registered; not routable until cutover")
13278 })
13279 .collect();
13280 assert_eq!(
13281 new_admissions.len(),
13282 1,
13283 "unexpected admission logs: {captured}"
13284 );
13285 assert!(new_admissions[0].contains("module_id=aft"));
13286 assert!(new_admissions[0].contains("connection_id=40"));
13287 assert!(
13288 new_admissions[0].contains("swap candidate registered; not routable until cutover")
13289 );
13290 assert!(new_admissions[0].contains("ready=true"));
13291
13292 let old_admissions: Vec<_> = captured
13293 .lines()
13294 .filter(|line| {
13295 line.contains("module registered module_id=aft ")
13296 && line.contains("connection_id=30")
13297 })
13298 .collect();
13299 assert_eq!(
13300 old_admissions.len(),
13301 1,
13302 "unexpected incumbent admission: {captured}"
13303 );
13304
13305 let old_ends: Vec<_> = captured
13306 .lines()
13307 .filter(|line| {
13308 line.contains("connection_id=30") && line.contains("module registration ended")
13309 })
13310 .collect();
13311 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
13312 assert!(old_ends[0].contains("module_id=aft"));
13313 assert!(old_ends[0].contains("connection_id=30"));
13314 assert!(old_ends[0].contains("reason=replaced"));
13315 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
13316 assert_eq!(
13317 captured
13318 .lines()
13319 .filter(|line| line.contains("module registration promoted"))
13320 .count(),
13321 1,
13322 "unexpected promotion logs: {captured}"
13323 );
13324 }
13325
13326 #[tokio::test]
13332 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
13333 let mut swap = swap_with_incumbent().await;
13334 let handler = swap.handler.clone();
13335
13336 let cotenant = ConnectionId::new(31);
13338 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
13339 let (cotenant_task, cotenant_bind) = relay_route_open(
13340 &handler,
13341 cotenant,
13342 &cotenant_ctx.egress,
13343 &mut swap.incumbent_rx,
13344 100,
13345 "aft",
13346 "swap-cotenant",
13347 )
13348 .await;
13349 handler
13350 .handle_control_frame(
13351 &swap.incumbent_ctx,
13352 route_bind_ack(cotenant_bind.header.corr),
13353 )
13354 .await
13355 .unwrap();
13356 assert!(cotenant_task.await.unwrap().is_empty());
13357 let (cotenant_channel, cotenant_epoch) =
13358 published_route(&cotenant_rx.recv().await.unwrap());
13359
13360 let caller = ConnectionId::new(32);
13362 let (caller_ctx, mut caller_rx) = route_ctx(caller);
13363 let (caller_task, caller_bind) = relay_route_open(
13364 &handler,
13365 caller,
13366 &caller_ctx.egress,
13367 &mut swap.incumbent_rx,
13368 101,
13369 "aft",
13370 "swap-caller",
13371 )
13372 .await;
13373 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
13374
13375 register_candidate(&swap, None);
13376 cutover(&swap);
13377
13378 let ack = handler
13380 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
13381 .await;
13382 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
13383 if module_loop_error.is_some() {
13384 handler.cleanup_connection(INCUMBENT).unwrap();
13387 }
13388
13389 assert!(
13391 cotenant_rx.try_recv().is_err(),
13392 "the co-tenant route on the incumbent was torn down by one late ack: \
13393 {module_loop_error:?}"
13394 );
13395 assert!(matches!(
13396 swap.forwarding
13397 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
13398 .unwrap(),
13399 DataRoute::Client(DataRouteState::Bound(_))
13400 ));
13401 assert_eq!(module_loop_error, None);
13402 assert!(swap
13403 .registry
13404 .get_module_by_connection(INCUMBENT)
13405 .unwrap()
13406 .is_some());
13407
13408 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
13410 .await
13411 .expect("the incumbent is told to drop the abandoned binding")
13412 .unwrap()
13413 .frame;
13414 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13415 assert_eq!(goodbye.header.channel, abandoned_channel);
13416 assert_eq!(goodbye.header.epoch, abandoned_epoch);
13417 assert!(swap.incumbent_rx.try_recv().is_err());
13418
13419 let response = caller_task.await.unwrap();
13421 assert_eq!(response.len(), 1);
13422 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
13423 assert!(caller_rx.try_recv().is_err());
13424
13425 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
13429 let counters = handler.counters().snapshot();
13430 assert_eq!(
13431 keyed_total(&counters, "route_open_accepted_by_principal"),
13432 1
13433 );
13434 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
13435 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
13436 }
13437
13438 #[tokio::test]
13442 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
13443 let mut swap = swap_with_incumbent().await;
13444 register_candidate(&swap, None);
13445 let incumbent = cutover(&swap);
13446 swap.forwarding
13447 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
13448 .unwrap()
13449 .expect("the incumbent is still registered");
13450
13451 let client = ConnectionId::new(33);
13452 let (client_ctx, mut client_rx) = route_ctx(client);
13453 let route_handler = swap.handler.clone();
13454 let open_ctx = RouteCtx {
13455 connection_id: client,
13456 egress: client_ctx.egress.clone(),
13457 };
13458 let mut route_task = tokio::spawn(async move {
13459 route_handler
13460 .handle_control_frame(
13461 &open_ctx,
13462 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
13463 )
13464 .await
13465 .unwrap()
13466 });
13467 let bind = tokio::select! {
13468 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
13469 response = &mut route_task => {
13470 let response = response.unwrap();
13471 panic!(
13472 "post-cutover route.open was refused instead of relayed to the candidate: {}",
13473 parse_error(&response[0])["code"]
13474 );
13475 }
13476 };
13477 swap.handler
13478 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
13479 .await
13480 .unwrap();
13481 assert!(route_task.await.unwrap().is_empty());
13482 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
13483 match swap
13484 .forwarding
13485 .lookup_data_route(client, channel, epoch)
13486 .unwrap()
13487 {
13488 DataRoute::Client(DataRouteState::Bound(route)) => {
13489 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
13490 }
13491 other => panic!("expected a bound route on the candidate, got {other:?}"),
13492 }
13493 assert!(swap.incumbent_rx.try_recv().is_err());
13494 }
13495
13496 #[tokio::test]
13501 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
13502 let swap = swap_with_incumbent().await;
13503 register_candidate(&swap, Some(false));
13504 let update = Frame::build(
13505 FrameType::Request,
13506 control_flags(),
13507 0,
13508 0,
13509 55,
13510 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13511 provides: manifest("aft", PROTOCOL_VERSION).provides,
13512 capabilities: None,
13513 ready: Some(true),
13514 })
13515 .unwrap(),
13516 )
13517 .unwrap();
13518
13519 let replies = swap
13520 .handler
13521 .handle_control_frame(&swap.candidate_ctx, update)
13522 .await
13523 .unwrap();
13524
13525 assert_eq!(replies.len(), 1);
13526 assert_eq!(
13527 replies[0].header.ty,
13528 FrameType::Response,
13529 "candidate catalog.update was refused: {:?}",
13530 serde_json::from_slice::<Value>(&replies[0].body).ok()
13531 );
13532 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
13533 assert_eq!(
13534 swap.registry
13535 .get_module("aft")
13536 .unwrap()
13537 .unwrap()
13538 .connection_id,
13539 INCUMBENT
13540 );
13541 }
13542 }
13543
13544 mod swap_admission {
13548 use super::*;
13549
13550 const INCUMBENT_NONCE: &str = "incumbent-nonce";
13551 const CANDIDATE_NONCE: &str = "candidate-nonce";
13552
13553 fn handler_with_incumbent(
13554 module_id: &str,
13555 reserved: bool,
13556 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
13557 let registry = Arc::new(Registry::default());
13558 let supervisor = SupervisorHandle::new();
13559 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
13560 if reserved {
13561 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
13562 }
13563 let handler =
13564 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
13565 let incumbent = handler
13566 .handle_control(
13567 ConnectionId::new(1),
13568 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
13569 )
13570 .unwrap();
13571 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
13572 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
13573 (registry, supervisor, handler)
13574 }
13575
13576 #[test]
13582 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
13583 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
13584
13585 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
13586 let replies = handler
13587 .handle_control(
13588 ConnectionId::new(connection),
13589 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
13590 )
13591 .unwrap();
13592 assert_eq!(replies[0].header.ty, FrameType::Error);
13593 assert_eq!(
13594 parse_error(&replies[0])["code"],
13595 "swap_token_invalid",
13596 "nonce {nonce:?}"
13597 );
13598 }
13599 assert!(registry.get_candidate("aft").unwrap().is_none());
13600 assert_eq!(
13601 registry.get_module("aft").unwrap().unwrap().connection_id,
13602 ConnectionId::new(1)
13603 );
13604
13605 let admitted = handler
13608 .handle_control(
13609 ConnectionId::new(4),
13610 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
13611 )
13612 .unwrap();
13613 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
13614 assert_eq!(
13615 registry
13616 .get_candidate("aft")
13617 .unwrap()
13618 .unwrap()
13619 .connection_id,
13620 ConnectionId::new(4)
13621 );
13622 assert_eq!(
13623 registry.get_module("aft").unwrap().unwrap().connection_id,
13624 ConnectionId::new(1),
13625 "the candidate must not take the active slot"
13626 );
13627 let replayed = handler
13628 .handle_control(
13629 ConnectionId::new(5),
13630 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
13631 )
13632 .unwrap();
13633 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
13634
13635 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
13639 let squatter = handler
13640 .handle_control(
13641 ConnectionId::new(6),
13642 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
13643 )
13644 .unwrap();
13645 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
13646 assert!(
13647 registry.get_module("aft").unwrap().is_none(),
13648 "a squatter took the active slot of an id being swapped"
13649 );
13650 }
13651
13652 #[test]
13657 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
13658 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
13659
13660 let replies = handler
13661 .handle_control(
13662 ConnectionId::new(2),
13663 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
13664 )
13665 .unwrap();
13666
13667 assert_eq!(
13668 replies[0].header.ty,
13669 FrameType::HelloAck,
13670 "reserved candidate refused: {:?}",
13671 serde_json::from_slice::<Value>(&replies[0].body).ok()
13672 );
13673 assert_eq!(
13674 registry
13675 .get_candidate("vault")
13676 .unwrap()
13677 .unwrap()
13678 .connection_id,
13679 ConnectionId::new(2)
13680 );
13681 }
13682
13683 #[test]
13686 fn without_an_open_swap_the_ordinary_gates_decide() {
13687 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
13688 supervisor.close_swap("vault");
13689
13690 let candidate = handler
13691 .handle_control(
13692 ConnectionId::new(2),
13693 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
13694 )
13695 .unwrap();
13696 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
13697 let duplicate = handler
13698 .handle_control(
13699 ConnectionId::new(3),
13700 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
13701 )
13702 .unwrap();
13703 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13704 assert!(registry.get_candidate("vault").unwrap().is_none());
13705 }
13706 }
13707
13708 mod scopes {
13712 mod activity {
13713 include!("activity_scope_tests.rs");
13714 }
13715 mod delta {
13716 include!("scope_handler_delta_tests.rs");
13717 }
13718 mod call_modes {
13719 include!("scope_call_mode_tests.rs");
13720 }
13721
13722 use subc_protocol::scope::{
13723 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
13724 ScopeStatus,
13725 };
13726
13727 use super::*;
13728
13729 const OWNER: &str = "prefrontal-core";
13730
13731 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
13732 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
13733 }
13734
13735 async fn call(
13736 handler: &ControlHandler,
13737 ctx: &RouteCtx,
13738 request: &ModuleControlRequestFromModule,
13739 ) -> Frame {
13740 let body = serde_json::to_vec(request).unwrap();
13741 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
13742 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
13743 assert_eq!(replies.len(), 1, "{replies:?}");
13744 replies.pop().unwrap()
13745 }
13746
13747 async fn sync(
13748 handler: &ControlHandler,
13749 ctx: &RouteCtx,
13750 generation: u64,
13751 scopes: Vec<ScopeRecord>,
13752 ) -> Result<ModuleControlResponseToModule, String> {
13753 let reply = call(
13754 handler,
13755 ctx,
13756 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
13757 )
13758 .await;
13759 match reply.header.ty {
13760 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
13761 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
13762 }
13763 }
13764
13765 async fn describe(
13766 handler: &ControlHandler,
13767 ctx: &RouteCtx,
13768 owner: &str,
13769 scope_ref: &str,
13770 ) -> ModuleControlResponseToModule {
13771 let reply = call(
13772 handler,
13773 ctx,
13774 &ModuleControlRequestFromModule::ScopeDescribe {
13775 owner: Principal::Reserved {
13776 module_id: owner.to_string(),
13777 },
13778 scope_ref: scope_ref.to_string(),
13779 },
13780 )
13781 .await;
13782 assert_eq!(
13783 reply.header.ty,
13784 FrameType::Response,
13785 "{:?}",
13786 parse_error(&reply)
13787 );
13788 serde_json::from_slice(&reply.body).unwrap()
13789 }
13790
13791 async fn module(
13793 handler: &ControlHandler,
13794 connection: u64,
13795 module_id: &str,
13796 nonce: Option<&str>,
13797 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
13798 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
13799 hello_via_sink(
13800 handler,
13801 &ctx,
13802 &mut rx,
13803 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
13804 )
13805 .await;
13806 (ctx, rx)
13807 }
13808
13809 #[tokio::test]
13812 async fn a_client_connection_cannot_sync_or_describe() {
13813 let handler = ControlHandler::new(Arc::new(Registry::default()));
13814 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
13815 for request in [
13816 ModuleControlRequestFromModule::ScopeSync {
13817 generation: 1,
13818 scopes: vec![head("s", 1)],
13819 },
13820 ModuleControlRequestFromModule::ScopeDescribe {
13821 owner: Principal::Direct,
13822 scope_ref: "s".to_string(),
13823 },
13824 ] {
13825 let reply = call(&handler, &ctx, &request).await;
13826 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
13827 }
13828 assert!(
13829 !handler
13830 .scopes
13831 .read()
13832 .unwrap()
13833 .describe(
13834 &Principal::Reserved {
13835 module_id: OWNER.to_string()
13836 },
13837 "s"
13838 )
13839 .owner_synced
13840 );
13841 }
13842
13843 #[tokio::test]
13846 async fn a_module_without_a_supervised_launch_cannot_sync() {
13847 let handler = ControlHandler::new(Arc::new(Registry::default()));
13848 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
13849 assert_eq!(
13850 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
13851 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13852 );
13853 }
13854
13855 #[tokio::test]
13856 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
13857 let supervisor = SupervisorHandle::new();
13858 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13859 let handler = ControlHandler::new(Arc::new(Registry::default()))
13860 .with_supervisor(supervisor.clone());
13861 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
13862 sync(&handler, &incumbent, 1, vec![head("s", 1)])
13863 .await
13864 .expect("the current launch syncs");
13865
13866 supervisor.open_swap(OWNER, "n2".to_string());
13869 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
13870 assert_eq!(
13871 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
13872 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13873 );
13874 sync(&handler, &incumbent, 2, vec![head("s", 1)])
13875 .await
13876 .expect("the serving owner syncs during the swap");
13877
13878 supervisor.close_swap(OWNER);
13881 assert_eq!(
13882 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
13883 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13884 );
13885 sync(&handler, &incumbent, 3, vec![head("s", 1)])
13886 .await
13887 .expect("the serving owner syncs after the rollback");
13888 handler.cleanup_connection(candidate.connection_id).unwrap();
13889
13890 supervisor.open_swap(OWNER, "n3".to_string());
13895 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
13896 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
13897 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
13898 .await
13899 .expect("the promoted launch takes authority");
13900 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
13901 panic!("unexpected reply {reply:?}");
13902 };
13903 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
13904 assert_eq!(
13905 sync(&handler, &incumbent, 4, Vec::new()).await,
13906 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13907 );
13908 }
13909
13910 #[tokio::test]
13913 async fn closing_the_authority_connection_frees_sync_authority() {
13914 let supervisor = SupervisorHandle::new();
13915 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13916 let handler = ControlHandler::new(Arc::new(Registry::default()))
13917 .with_supervisor(supervisor.clone());
13918 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
13919 sync(&handler, &first, 10, vec![head("s", 1)])
13920 .await
13921 .unwrap();
13922 handler.cleanup_connection(first.connection_id).unwrap();
13923
13924 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
13925 sync(&handler, &second, 1, vec![head("s", 1)])
13926 .await
13927 .expect("the next connection takes the released authority");
13928 }
13929
13930 #[tokio::test]
13931 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
13932 let supervisor = SupervisorHandle::new();
13933 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13934 let handler =
13935 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
13936 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
13937 sync(&handler, &first, 10, vec![head("s", 1)])
13938 .await
13939 .unwrap();
13940 handler
13941 .handle_control_frame(
13942 &first,
13943 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
13944 )
13945 .await
13946 .unwrap();
13947 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
13948 sync(&handler, &second, 1, vec![head("s", 1)])
13949 .await
13950 .expect("GOODBYE releases authority even if the old socket remains open");
13951 }
13952
13953 #[tokio::test]
13954 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
13955 let registry = Arc::new(Registry::default());
13956 let supervisor_handle = SupervisorHandle::new();
13957 let supervisor =
13958 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
13959 .with_handle(supervisor_handle.clone())
13960 .with_daemon_incarnation("incarnation-7".to_string());
13961 supervisor
13964 .supervise_configured(
13965 ModuleSpec {
13966 module_id: OWNER.to_string(),
13967 program: PathBuf::from("/nonexistent/prefrontal-core"),
13968 args: Vec::new(),
13969 env: Vec::new(),
13970 reserved: false,
13971 reserved_prefixes: Vec::new(),
13972 protocol: ModuleProtocol::Subc,
13973 overlap: Default::default(),
13974 },
13975 false,
13976 )
13977 .unwrap();
13978 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
13979 let handler =
13980 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13981 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
13982
13983 let ModuleControlResponseToModule::ScopeDescribe {
13985 status,
13986 daemon_incarnation,
13987 owner_synced,
13988 owner_configured,
13989 scope,
13990 ..
13991 } = describe(&handler, &reader, OWNER, "s").await
13992 else {
13993 panic!("not a describe reply");
13994 };
13995 assert_eq!(status, ScopeStatus::NotLive);
13996 assert_eq!(daemon_incarnation, "incarnation-7");
13997 assert!(!owner_synced);
13998 assert!(owner_configured);
13999 assert!(scope.is_none());
14000
14001 let ModuleControlResponseToModule::ScopeDescribe {
14004 status,
14005 owner_configured,
14006 ..
14007 } = describe(&handler, &reader, "ghost", "s").await
14008 else {
14009 panic!("not a describe reply");
14010 };
14011 assert_eq!(status, ScopeStatus::NotLive);
14012 assert!(!owner_configured);
14013
14014 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
14016 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
14017 let ModuleControlResponseToModule::ScopeDescribe {
14018 status,
14019 scope_epoch,
14020 owner_synced,
14021 scope,
14022 ..
14023 } = describe(&handler, &reader, OWNER, "s").await
14024 else {
14025 panic!("not a describe reply");
14026 };
14027 assert_eq!(status, ScopeStatus::Live);
14028 assert_eq!(scope_epoch, Some(4));
14029 assert!(owner_synced);
14030 let stamp = scope.expect("a live scope carries its stamp");
14031 assert!(
14032 stamp.owner_authorized,
14033 "prefrontal-core is the default authority"
14034 );
14035 assert_eq!(stamp.kind, ScopeKind::Head);
14036 }
14037
14038 #[tokio::test]
14039 async fn scope_authority_owners_decides_owner_authorized() {
14040 let supervisor = SupervisorHandle::new();
14041 supervisor.set_spawn_nonce("broca", "b1".to_string());
14042 let handler = ControlHandler::new(Arc::new(Registry::default()))
14043 .with_supervisor(supervisor)
14044 .with_scope_authority_owners(vec!["broca".to_string()]);
14045 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
14046 let mut gated = head("s", 1);
14047 gated.attributes.agent_id = Some("agent".to_string());
14048 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
14049 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
14050 describe(&handler, &broca, "broca", "s").await
14051 else {
14052 panic!("not a describe reply");
14053 };
14054 assert!(scope.unwrap().owner_authorized);
14055 }
14056
14057 #[tokio::test]
14061 async fn scope_ops_and_the_scopes_capability_are_advertised() {
14062 let handler = ControlHandler::new(Arc::new(Registry::default()));
14063 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
14064 let ack = hello_via_sink(
14065 &handler,
14066 &ctx,
14067 &mut rx,
14068 hello_frame("m", PROTOCOL_VERSION, 1),
14069 )
14070 .await;
14071 let ack = parse_ack(&ack);
14072 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
14073 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
14074 }
14075 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
14076
14077 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
14078 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
14079 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
14080 let reply = handler
14081 .handle_control_frame(&client, frame)
14082 .await
14083 .unwrap()
14084 .pop()
14085 .unwrap();
14086 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14087 serde_json::from_slice(&reply.body).unwrap()
14088 else {
14089 panic!("not a server.describe reply");
14090 };
14091 assert!(
14092 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
14093 "{capabilities:?}"
14094 );
14095 }
14096
14097 const PLEXUS: &str = "plexus";
14100 const OTHER: &str = "other";
14101 const AFT: &str = "aft";
14102 const BROCA: &str = "broca";
14103 const MAGIC: &str = "magic-context";
14104
14105 fn nonce(module_id: &str) -> String {
14106 format!("nonce-{module_id}")
14107 }
14108
14109 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
14110 let (tx, rx) = mpsc::channel(64);
14111 (
14112 RouteCtx {
14113 connection_id: ConnectionId::new(connection),
14114 egress: FrameSink::new(tx),
14115 },
14116 rx,
14117 )
14118 }
14119
14120 struct Rig {
14124 handler: ControlHandler,
14125 forwarding: Arc<ForwardingTable>,
14126 owner: RouteCtx,
14127 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14128 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
14129 generation: u64,
14130 next_connection: u64,
14131 _supervisor: Supervisor,
14132 }
14133
14134 async fn rig() -> Rig {
14135 rig_with_flow_support(true).await
14136 }
14137
14138 async fn rig_with_flow_support(flow_support: bool) -> Rig {
14139 rig_with_scope_capabilities(flow_support, false).await
14140 }
14141
14142 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
14143 let mut provides = Vec::new();
14144 if flow_support {
14145 provides.push("flow-scopes/v1");
14146 }
14147 if run_support {
14148 provides.push("agent-run-scopes/v1");
14149 }
14150 rig_with_target_capabilities(&provides).await
14151 }
14152
14153 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
14154 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
14155 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
14156 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
14159 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
14160 Frame::build(
14161 FrameType::Hello,
14162 control_flags(),
14163 0,
14164 0,
14165 connection,
14166 serde_json::to_vec(&body).unwrap(),
14167 )
14168 .unwrap()
14169 }
14170
14171 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
14172 let registry = Arc::new(Registry::default());
14173 let forwarding = Arc::new(ForwardingTable::default());
14174 let supervisor_handle = SupervisorHandle::new();
14175 let supervisor =
14176 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14177 .with_handle(supervisor_handle.clone());
14178 supervisor
14179 .supervise_configured(
14180 ModuleSpec {
14181 module_id: OWNER.to_string(),
14182 program: PathBuf::from("/nonexistent/prefrontal-core"),
14183 args: Vec::new(),
14184 env: Vec::new(),
14185 reserved: false,
14186 reserved_prefixes: Vec::new(),
14187 protocol: ModuleProtocol::Subc,
14188 overlap: Default::default(),
14189 },
14190 false,
14191 )
14192 .unwrap();
14193 for module_id in [OWNER, AFT, BROCA, MAGIC] {
14194 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
14195 }
14196 let handler =
14197 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14198 .with_supervisor(supervisor_handle);
14199 let (owner, mut owner_rx) = wide_ctx(1);
14200 hello_via_sink(
14201 &handler,
14202 &owner,
14203 &mut owner_rx,
14204 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
14205 )
14206 .await;
14207 let mut modules = BTreeMap::new();
14208 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
14209 let (ctx, mut rx) = wide_ctx(connection);
14210 let hello = hello_with_capabilities(module_id, connection, provides);
14211 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
14212 modules.insert(module_id.to_string(), (ctx, rx));
14213 }
14214 Rig {
14215 handler,
14216 forwarding,
14217 owner,
14218 _owner_rx: owner_rx,
14219 modules,
14220 generation: 0,
14221 next_connection: 100,
14222 _supervisor: supervisor,
14223 }
14224 }
14225
14226 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
14227 ScopeCarrier::new(Principal::Reserved {
14228 module_id: module_id.to_string(),
14229 })
14230 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
14231 }
14232
14233 fn session(scope_epoch: u64) -> ScopeRecord {
14236 let mut record = head("s", scope_epoch);
14237 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
14238 record.attributes.agent_id = Some("agent-1".to_string());
14239 record.attributes.delegates = true;
14240 record
14241 }
14242
14243 impl Rig {
14244 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
14245 self.generation += 1;
14246 sync(&self.handler, &self.owner, self.generation, scopes)
14247 .await
14248 .expect("the owner's sync is accepted");
14249 }
14250
14251 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
14252 ScopeSelector {
14253 owner: Principal::Reserved {
14254 module_id: OWNER.to_string(),
14255 },
14256 scope_ref: scope_ref.to_string(),
14257 scope_epoch,
14258 }
14259 }
14260
14261 fn open_frame(
14262 &mut self,
14263 opener: Option<&str>,
14264 target: &str,
14265 scope: Option<ScopeSelector>,
14266 ) -> (
14267 RouteCtx,
14268 mpsc::Receiver<crate::router::OutboundFrame>,
14269 Frame,
14270 ) {
14271 self.next_connection += 1;
14272 let (ctx, rx) = wide_ctx(self.next_connection);
14273 let root = unique_project_root("scoped-open");
14274 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
14275 target: RouteTarget::ToolProvider {
14276 module_id: target.to_string(),
14277 },
14278 identity: BindIdentity::new(
14279 root.path().to_path_buf(),
14280 "unit".to_string(),
14281 "session".to_string(),
14282 ),
14283 consumer_identity: opener.map(|module_id| ConsumerIdentity {
14284 module_id: module_id.to_string(),
14285 launch_nonce: nonce(module_id),
14286 }),
14287 consumer_capabilities: None,
14288 role_versions: None,
14289 admission_facts: None,
14290 scope,
14291 })
14292 .unwrap();
14293 let frame = Frame::build(
14294 FrameType::Request,
14295 control_flags(),
14296 0,
14297 0,
14298 self.next_connection,
14299 body,
14300 )
14301 .unwrap();
14302 (ctx, rx, frame)
14303 }
14304
14305 async fn refused(
14307 &mut self,
14308 opener: Option<&str>,
14309 target: &str,
14310 scope: Option<ScopeSelector>,
14311 ) -> String {
14312 self.refusal_body(opener, target, scope).await["code"]
14313 .as_str()
14314 .unwrap()
14315 .to_string()
14316 }
14317
14318 async fn refusal_body(
14319 &mut self,
14320 opener: Option<&str>,
14321 target: &str,
14322 scope: Option<ScopeSelector>,
14323 ) -> Value {
14324 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
14325 let replies = tokio::time::timeout(
14326 Duration::from_secs(2),
14327 self.handler.handle_control_frame(&ctx, frame),
14328 )
14329 .await
14330 .expect("the open must be refused before waiting for a bind ack")
14331 .unwrap();
14332 assert_eq!(replies.len(), 1, "{replies:?}");
14333 assert_eq!(replies[0].header.ty, FrameType::Error);
14334 let (_, module_rx) = self.modules.get_mut(target).unwrap();
14335 assert!(
14336 module_rx.try_recv().is_err(),
14337 "a refused open relays nothing"
14338 );
14339 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
14340 parse_error(&replies[0])
14341 }
14342
14343 async fn relayed(
14345 &mut self,
14346 opener: Option<&str>,
14347 target: &str,
14348 scope: Option<ScopeSelector>,
14349 ) -> Relayed {
14350 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
14351 let handler = self.handler.clone();
14352 let task_ctx = ctx.clone();
14353 let task = tokio::spawn(async move {
14354 handler
14355 .handle_control_frame(&task_ctx, frame)
14356 .await
14357 .unwrap()
14358 });
14359 let (_, module_rx) = self.modules.get_mut(target).unwrap();
14360 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
14361 .await
14362 .expect("the target receives the relayed route.bind")
14363 .unwrap()
14364 .frame;
14365 Relayed {
14366 target: target.to_string(),
14367 client: ctx,
14368 client_rx: rx,
14369 task,
14370 bind,
14371 }
14372 }
14373
14374 async fn ack(&self, relayed: &Relayed) {
14375 let (module, _) = &self.modules[&relayed.target];
14376 self.handler
14377 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
14378 .await
14379 .unwrap();
14380 }
14381
14382 async fn bound(
14384 &mut self,
14385 opener: Option<&str>,
14386 target: &str,
14387 scope: Option<ScopeSelector>,
14388 ) -> Bound {
14389 let relayed = self.relayed(opener, target, scope).await;
14390 self.ack(&relayed).await;
14391 let Relayed {
14392 target,
14393 client,
14394 mut client_rx,
14395 task,
14396 bind,
14397 } = relayed;
14398 assert!(
14399 task.await.unwrap().is_empty(),
14400 "the open is answered by commit"
14401 );
14402 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
14403 Bound {
14404 target,
14405 client,
14406 client_rx,
14407 channel,
14408 epoch,
14409 bind,
14410 }
14411 }
14412
14413 fn live(&self, route: &Bound) -> bool {
14414 matches!(
14415 self.forwarding
14416 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
14417 .unwrap(),
14418 DataRoute::Client(DataRouteState::Bound(_))
14419 )
14420 }
14421 }
14422
14423 struct Relayed {
14424 target: String,
14425 client: RouteCtx,
14426 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14427 task: tokio::task::JoinHandle<Vec<Frame>>,
14428 bind: Frame,
14429 }
14430
14431 struct Bound {
14432 target: String,
14433 client: RouteCtx,
14434 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14435 channel: u16,
14436 epoch: u32,
14437 bind: Frame,
14438 }
14439
14440 impl Bound {
14441 fn closed_reason(&mut self) -> RouteCloseReason {
14444 let mut reason = None;
14445 let mut goodbye = false;
14446 while let Ok(outbound) = self.client_rx.try_recv() {
14447 let frame = outbound.frame;
14448 match frame.header.ty {
14449 FrameType::Goodbye => {
14450 assert_eq!(
14451 (frame.header.channel, frame.header.epoch),
14452 (self.channel, self.epoch)
14453 );
14454 goodbye = true;
14455 }
14456 FrameType::Push => {
14457 let ClientControlPush::RouteClosed {
14458 reason: r,
14459 module_id,
14460 ..
14461 } = serde_json::from_slice(&frame.body).unwrap()
14462 else {
14463 panic!("unexpected push");
14464 };
14465 assert_eq!(module_id, self.target);
14466 reason = Some(r);
14467 }
14468 other => panic!("unexpected frame {other:?}"),
14469 }
14470 }
14471 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
14472 reason.expect("the client is told why the route closed")
14473 }
14474
14475 fn untouched(&mut self) -> bool {
14476 self.client_rx.try_recv().is_err()
14477 }
14478
14479 fn stamp(&self) -> Option<ScopeStamp> {
14480 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
14481 ModuleControlRequest::RouteBind { scope, .. } => scope,
14482 other => panic!("expected a route.bind, got {other:?}"),
14483 }
14484 }
14485 }
14486
14487 #[tokio::test]
14488 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
14489 ) {
14490 let mut rig = rig().await;
14491 let mut record = session(1);
14492 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
14493 record.child_owners = vec![Principal::Reserved {
14494 module_id: MAGIC.to_string(),
14495 }];
14496 rig.sync(vec![record]).await;
14497 let scope = || Some(rig_selector("s", Some(1)));
14498
14499 rig.bound(Some(OWNER), PLEXUS, scope()).await;
14502 rig.bound(Some(AFT), OTHER, scope()).await;
14503 rig.bound(Some(BROCA), PLEXUS, scope()).await;
14504
14505 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
14509 assert_eq!(
14510 rig.refused(opener, target, scope()).await,
14511 error_codes::SCOPE_NOT_CARRIER,
14512 "{opener:?} -> {target}"
14513 );
14514 }
14515 }
14516
14517 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
14518 ScopeSelector {
14519 owner: Principal::Reserved {
14520 module_id: OWNER.to_string(),
14521 },
14522 scope_ref: scope_ref.to_string(),
14523 scope_epoch,
14524 }
14525 }
14526
14527 #[tokio::test]
14528 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
14529 let mut rig = rig().await;
14530 rig.sync(vec![session(1)]).await;
14531 for opener in [OWNER, AFT] {
14532 assert_eq!(
14533 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
14534 .await,
14535 error_codes::SCOPE_EPOCH_REQUIRED,
14536 "{opener}"
14537 );
14538 }
14539 }
14540
14541 #[tokio::test]
14542 async fn admission_separates_not_synced_not_live_and_ended() {
14543 let mut rig = rig().await;
14544 let code = rig
14546 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14547 .await;
14548 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
14549 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
14550
14551 let ghost = ScopeSelector {
14553 owner: Principal::Reserved {
14554 module_id: "ghost".to_string(),
14555 },
14556 scope_ref: "s".to_string(),
14557 scope_epoch: Some(1),
14558 };
14559 assert_eq!(
14560 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
14561 error_codes::SCOPE_NOT_LIVE
14562 );
14563
14564 rig.sync(vec![session(2)]).await;
14565 assert_eq!(
14566 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
14567 .await,
14568 error_codes::SCOPE_NOT_LIVE
14569 );
14570 for epoch in [1, 3] {
14571 assert_eq!(
14572 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
14573 .await,
14574 error_codes::SCOPE_ENDED,
14575 "epoch {epoch}"
14576 );
14577 }
14578 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
14580 .await;
14581 }
14582
14583 #[tokio::test]
14584 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
14585 let mut rig = rig().await;
14586 rig.sync(vec![session(1)]).await;
14587 let route = rig
14588 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14589 .await;
14590 let stamp = route.stamp().expect("a scoped bind carries the stamp");
14591 assert_eq!(stamp.scope_ref, "s");
14592 assert_eq!(stamp.scope_epoch, 1);
14593 assert_eq!(stamp.kind, ScopeKind::Head);
14594 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
14595 assert!(stamp.attributes.delegates);
14596 assert!(stamp.owner_authorized);
14597 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14598 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
14599
14600 let (broca, mut broca_rx) = wide_ctx(50);
14603 hello_via_sink(
14604 &rig.handler,
14605 &broca,
14606 &mut broca_rx,
14607 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
14608 )
14609 .await;
14610 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
14611 .await
14612 .unwrap();
14613 let own = ScopeSelector {
14614 owner: Principal::Reserved {
14615 module_id: BROCA.to_string(),
14616 },
14617 scope_ref: "b".to_string(),
14618 scope_epoch: Some(1),
14619 };
14620 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
14621 assert!(!route.stamp().unwrap().owner_authorized);
14622 }
14623
14624 #[tokio::test]
14625 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
14626 let mut rig = rig().await;
14627 let mut record = head("s", 1);
14628 record.carriers = vec![carrier(AFT, None)];
14629 let flow_id = "Flow:run-7/step_2!~";
14630 record.attributes.flow_id = Some(flow_id.to_string());
14631 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
14632 .await
14633 .unwrap();
14634 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
14635 panic!("not a sync reply");
14636 };
14637 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
14638 let route = rig
14639 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14640 .await;
14641 assert!(rig.live(&route), "the stamped bind committed");
14642 let stamp = route.stamp().expect("a flow scope carries a stamp");
14643 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
14644 assert_eq!(stamp.attributes.agent_id, None);
14645 assert!(!stamp.attributes.delegates);
14646 assert!(stamp.owner_authorized);
14647 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14648 assert_eq!(unscoped.stamp(), None);
14649 }
14650
14651 #[tokio::test]
14652 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
14653 let mut rig = rig_with_flow_support(false).await;
14654 let mut record = session(1);
14655 record.attributes.flow_id = Some("flow:7".to_string());
14656 rig.sync(vec![record]).await;
14657 for opener in [OWNER, AFT] {
14658 let body = rig
14659 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
14660 .await;
14661 assert_eq!(body["code"], "target_flow_unsupported");
14662 let message = body["message"].as_str().unwrap();
14663 for required in [PLEXUS, "flow-scopes/v1"] {
14664 assert!(message.contains(required), "{message}");
14665 }
14666 }
14667 }
14668
14669 #[tokio::test]
14670 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
14671 let mut rig = rig_with_flow_support(true).await;
14672 let mut record = session(1);
14673 record.attributes.flow_id = Some("flow:7".to_string());
14674 rig.sync(vec![record]).await;
14675 for opener in [OWNER, AFT] {
14676 let route = rig
14677 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
14678 .await;
14679 assert!(rig.live(&route));
14680 assert_eq!(
14681 route.stamp().unwrap().attributes.flow_id.as_deref(),
14682 Some("flow:7")
14683 );
14684 }
14685 }
14686
14687 #[tokio::test]
14688 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
14689 use std::future::Future;
14690
14691 let mut rig = rig_with_flow_support(true).await;
14692 let mut record = session(1);
14693 record.attributes.flow_id = Some("flow:7".to_string());
14694 rig.sync(vec![record]).await;
14695 let (client, mut client_rx, frame) =
14696 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
14697 for _ in 0..64 {
14700 client.egress.try_send(route_bind_ack(1)).unwrap();
14701 }
14702 let handler = rig.handler.clone();
14703 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
14704 std::future::poll_fn(|cx| {
14705 assert!(open.as_mut().poll(cx).is_pending());
14706 std::task::Poll::Ready(())
14707 })
14708 .await;
14709 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14710
14711 let old_connection = rig.modules[PLEXUS].0.connection_id;
14712 rig.handler.cleanup_connection(old_connection).unwrap();
14713 let (replacement, mut replacement_rx) = wide_ctx(200);
14714 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
14715 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
14716 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
14717 let hello = Frame::build(
14718 FrameType::Hello,
14719 control_flags(),
14720 0,
14721 0,
14722 200,
14723 serde_json::to_vec(&body).unwrap(),
14724 )
14725 .unwrap();
14726 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
14727
14728 client_rx.try_recv().unwrap();
14729 let replies = tokio::time::timeout(Duration::from_secs(2), open)
14730 .await
14731 .expect("the replacement is refused without waiting for a bind ack")
14732 .unwrap();
14733 assert_eq!(replies.len(), 1);
14734 let body = parse_error(&replies[0]);
14735 assert_eq!(body["code"], "target_flow_unsupported");
14736 for required in [PLEXUS, "flow-scopes/v1"] {
14737 assert!(body["message"].as_str().unwrap().contains(required));
14738 }
14739 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
14740 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
14741 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14742 assert!(rig
14743 .handler
14744 .registry
14745 .get_module_by_connection(replacement.connection_id)
14746 .unwrap()
14747 .is_some());
14748 }
14749
14750 #[tokio::test]
14751 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
14752 {
14753 let mut rig = rig_with_flow_support(false).await;
14754 rig.sync(vec![session(1)]).await;
14755 let route = rig
14756 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14757 .await;
14758 assert!(rig.live(&route));
14759 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
14760 let mut record = session(1);
14761 record.attributes.flow_id = Some("flow:7".to_string());
14762 rig.sync(vec![record]).await;
14763 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
14764 assert!(rig.live(&unscoped));
14765 assert_eq!(unscoped.stamp(), None);
14766 }
14767
14768 #[tokio::test]
14769 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
14770 let mut rig = rig().await;
14771 let mut record = session(1);
14772 record.attributes.flow_id = Some("flow:7".to_string());
14773 rig.sync(vec![record.clone()]).await;
14774 let before = rig.forwarding.published_scope_tag(OWNER, "s");
14775 let mut owner_route = rig
14776 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
14777 .await;
14778 let mut carrier_route = rig
14779 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
14780 .await;
14781 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14782 rig.sync(vec![record.clone()]).await;
14783 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
14784 assert!(rig.live(&owner_route) && owner_route.untouched());
14785 assert!(rig.live(&carrier_route) && carrier_route.untouched());
14786
14787 record.attributes.flow_id = Some("flow:8".to_string());
14788 rig.sync(vec![record]).await;
14789 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
14790 let before = before.unwrap();
14791 assert_eq!(after.scope_epoch, before.scope_epoch);
14792 assert!(after.version > before.version);
14793 for route in [&mut owner_route, &mut carrier_route] {
14794 assert!(!rig.live(route));
14795 assert_eq!(
14796 route.closed_reason(),
14797 RouteCloseReason::ScopeDelegationChanged
14798 );
14799 }
14800 assert!(rig.live(&unscoped) && unscoped.untouched());
14801 for target in [PLEXUS, OTHER] {
14804 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
14805 let goodbye = module_rx
14806 .try_recv()
14807 .expect("the provider sees the drain")
14808 .frame;
14809 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14810 assert!(module_rx.try_recv().is_err());
14811 }
14812 let rebound = rig
14813 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14814 .await;
14815 assert_eq!(
14816 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
14817 Some("flow:8")
14818 );
14819 }
14820
14821 #[tokio::test]
14826 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
14827 let mut rig = rig().await;
14828 rig.sync(vec![session(1)]).await;
14829 let mut cotenant = rig
14830 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
14831 .await;
14832
14833 let mut changed = session(1);
14834 changed.child_owners.push(Principal::Reserved {
14835 module_id: MAGIC.to_string(),
14836 });
14837 let mut ended = None;
14838 for (code, next) in [
14839 (error_codes::SCOPE_CHANGED, vec![changed]),
14840 (error_codes::SCOPE_ENDED, Vec::new()),
14841 ] {
14842 let relayed = rig
14843 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14844 .await;
14845 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
14846 ended = Some(next.is_empty());
14847 rig.sync(next).await;
14848 rig.ack(&relayed).await;
14849 let replies = relayed.task.await.unwrap();
14850 assert_eq!(replies.len(), 1, "{replies:?}");
14851 assert_eq!(parse_error(&replies[0])["code"], code);
14852 assert_eq!(
14853 subc_protocol::error_codes::is_retryable_route_open(code),
14854 code == error_codes::SCOPE_CHANGED
14855 );
14856 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14857 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14859 let mut goodbyes = Vec::new();
14862 while let Ok(outbound) = plexus_rx.try_recv() {
14863 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
14864 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
14865 }
14866 assert!(
14867 goodbyes.contains(&(bind_channel, bind_epoch)),
14868 "{goodbyes:?}"
14869 );
14870 assert!(rig
14871 .handler
14872 .registry
14873 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
14874 .unwrap()
14875 .is_some());
14876 }
14877 assert_eq!(ended, Some(true));
14878 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
14881 }
14882
14883 #[tokio::test]
14886 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
14887 struct Case {
14888 name: &'static str,
14889 change: fn(&mut ScopeRecord),
14890 closed: [Option<RouteCloseReason>; 4],
14893 }
14894 use RouteCloseReason::*;
14895 let cases = [
14896 Case {
14897 name: "a carrier entry removed",
14898 change: |r| {
14899 r.carriers.retain(|c| {
14900 c.principal
14901 != Principal::Reserved {
14902 module_id: AFT.to_string(),
14903 }
14904 })
14905 },
14906 closed: [None, Some(ScopeCarrierRemoved), None, None],
14907 },
14908 Case {
14909 name: "a target removed from a carrier",
14910 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
14911 closed: [None, None, None, Some(ScopeCarrierRemoved)],
14912 },
14913 Case {
14914 name: "a bare carrier narrowed to targets",
14915 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
14916 closed: [None, Some(ScopeCarrierRemoved), None, None],
14917 },
14918 Case {
14919 name: "delegates turned off",
14920 change: |r| r.attributes.delegates = false,
14921 closed: [Some(ScopeDelegationChanged); 4],
14922 },
14923 Case {
14924 name: "agent_id changed",
14925 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
14926 closed: [Some(ScopeDelegationChanged); 4],
14927 },
14928 Case {
14929 name: "a carrier added, child owners changed, the record re-sent",
14930 change: |r| {
14931 r.carriers.push(carrier(MAGIC, None));
14932 r.child_owners.push(Principal::Reserved {
14933 module_id: MAGIC.to_string(),
14934 });
14935 },
14936 closed: [None; 4],
14937 },
14938 Case {
14939 name: "a target added",
14940 change: |r| {
14941 r.carriers[1]
14942 .targets
14943 .as_mut()
14944 .unwrap()
14945 .push("third".to_string())
14946 },
14947 closed: [None; 4],
14948 },
14949 Case {
14950 name: "delegates turned on",
14951 change: |r| r.attributes.delegates = true,
14952 closed: [None; 4],
14953 },
14954 ];
14955 for case in cases {
14956 let mut rig = rig().await;
14957 rig.sync(vec![session(1)]).await;
14958 let scope = || Some(rig_selector("s", Some(1)));
14959 let mut routes = [
14960 rig.bound(Some(OWNER), PLEXUS, scope()).await,
14961 rig.bound(Some(AFT), PLEXUS, scope()).await,
14962 rig.bound(Some(BROCA), PLEXUS, scope()).await,
14963 rig.bound(Some(BROCA), OTHER, scope()).await,
14964 ];
14965 let mut record = session(1);
14966 (case.change)(&mut record);
14967 rig.sync(vec![record]).await;
14968 for (index, expected) in case.closed.iter().enumerate() {
14969 let route = &mut routes[index];
14970 match expected {
14971 Some(reason) => {
14972 assert!(!rig.live(route), "{}: route {index} still live", case.name);
14973 assert_eq!(
14974 route.closed_reason(),
14975 *reason,
14976 "{}: route {index}",
14977 case.name
14978 );
14979 }
14980 None => {
14981 assert!(rig.live(route), "{}: route {index} closed", case.name);
14982 assert!(
14983 route.untouched(),
14984 "{}: route {index} was told something",
14985 case.name
14986 );
14987 }
14988 }
14989 }
14990 }
14991 }
14992
14993 #[tokio::test]
14994 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
14995 for next in [Vec::new(), vec![session(2)]] {
14997 let mut rig = rig().await;
14998 rig.sync(vec![session(1)]).await;
14999 let mut route = rig
15000 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15001 .await;
15002 rig.sync(next).await;
15003 assert!(!rig.live(&route));
15004 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
15005 }
15006
15007 let mut rig = rig().await;
15010 let mut child = session(1);
15011 child.scope_ref = "child".to_string();
15012 child.kind = ScopeKind::Worker;
15013 child.parent = Some(ScopeParent::new(
15014 Principal::Reserved {
15015 module_id: OWNER.to_string(),
15016 },
15017 "s".to_string(),
15018 1,
15019 ));
15020 rig.sync(vec![session(1), child.clone()]).await;
15021 let mut child_route = rig
15022 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
15023 .await;
15024 assert_eq!(
15025 child_route.stamp().unwrap().parent_state,
15026 Some(ParentState::Linked)
15027 );
15028 rig.sync(vec![child]).await;
15029 assert!(!rig.live(&child_route));
15030 assert_eq!(
15031 child_route.closed_reason(),
15032 RouteCloseReason::ScopeParentEnded
15033 );
15034 }
15035
15036 #[tokio::test]
15037 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
15038 ) {
15039 let mut rig = rig().await;
15040 rig.sync(vec![session(1)]).await;
15041 let mut route = rig
15042 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15043 .await;
15044 let before = rig.forwarding.published_scope_tag(OWNER, "s");
15045 rig.sync(vec![session(1)]).await;
15046 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
15047 assert!(rig.live(&route) && route.untouched());
15048
15049 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
15053 .forwarding
15054 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15055 .unwrap()
15056 else {
15057 panic!("the route is bound");
15058 };
15059 binding.flow.acquire_tagged(9, false).await.unwrap();
15060 let mut widened = session(1);
15061 widened.carriers.push(carrier(MAGIC, None));
15062 rig.sync(vec![widened]).await;
15063 assert!(rig.live(&route) && route.untouched());
15064 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15065 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
15066 assert_eq!(binding.flow.in_flight(), 1);
15069 binding
15070 .flow
15071 .acquire_tagged(10, false)
15072 .await
15073 .expect("the flow is still open");
15074 }
15075
15076 #[tokio::test]
15079 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
15080 let mut rig = rig().await;
15081 rig.sync(vec![session(1)]).await;
15082 let mut on_incumbent = rig
15083 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15084 .await;
15085
15086 let (candidate, _candidate_rx) = wide_ctx(9);
15089 let registration = rig
15090 .handler
15091 .registry
15092 .register_candidate_with_control_ops(
15093 manifest(PLEXUS, PROTOCOL_VERSION),
15094 PROTOCOL_VERSION,
15095 candidate.connection_id,
15096 module_baseline_control_ops(),
15097 )
15098 .unwrap();
15099 rig.forwarding
15100 .register_candidate_module_connection(
15101 candidate.connection_id,
15102 PLEXUS.to_string(),
15103 PROTOCOL_VERSION,
15104 manifest_concurrency(®istration.manifest),
15105 candidate.egress.clone(),
15106 )
15107 .unwrap();
15108 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
15109 rig.handler
15110 .registry
15111 .promote_candidate(PLEXUS)
15112 .unwrap()
15113 .unwrap();
15114 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
15115
15116 rig.sync(Vec::new()).await;
15117 assert!(!rig.live(&on_incumbent));
15118 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
15119 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15120 let goodbye = incumbent_rx
15121 .try_recv()
15122 .expect("the superseded endpoint is told")
15123 .frame;
15124 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15125 }
15126 }
15127}
15128
15129#[cfg(test)]
15130mod concurrency_default_exposure_tests {
15131 use super::*;
15132
15133 fn hello_body(role_json: &str) -> Vec<u8> {
15134 format!(
15135 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
15136 )
15137 .into_bytes()
15138 }
15139
15140 fn manifest_from(body: &[u8]) -> ModuleManifest {
15141 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
15142 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
15143 .expect("manifest parses")
15144 }
15145
15146 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
15147
15148 #[test]
15149 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
15150 let body = hello_body(&format!(
15151 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
15152 ));
15153 let manifest = manifest_from(&body);
15154 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
15157 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
15158 }
15159
15160 #[test]
15161 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
15162 let body = hello_body(&format!(
15163 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
15164 ));
15165 let manifest = manifest_from(&body);
15166 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
15167 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
15168 }
15169
15170 #[test]
15171 fn non_management_roles_are_never_reported() {
15172 let body = hello_body(
15173 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
15174 );
15175 let manifest = manifest_from(&body);
15176 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
15177 }
15178}