1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
21 error_codes,
22 manifest::{
23 validate_hello_capability_grammar, validate_hello_event_declarations,
24 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
25 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
26 },
27 scope::{
28 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
29 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
30 SCOPE_SYNC_OP,
31 },
32 session::{
33 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
34 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
35 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
36 ROLE_VERSIONS_FIELD,
37 },
38 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
39 Priority, RouteTarget, PROTOCOL_VERSION,
40};
41use tokio::time::{timeout_at, Instant};
42use tracing::{debug, info, warn};
43
44use crate::{
45 capability_requirements::{
46 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
47 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
48 RuntimeModule,
49 },
50 daemon_config::RestartRequiredSection,
51 forwarding::{
52 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
53 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
54 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
55 },
56 observability::{
57 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
58 },
59 provenance::{
60 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
61 },
62 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
63 router::{RouteCtx, RouterError},
64 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
65 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
66 stderr_tail::{CaptureState, TailEntry},
67 supervise::{
68 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
69 SupervisorHandle, SwapHelloAdmission,
70 },
71 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
72};
73
74pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
80
81const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
82const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
83const CAP_PING_PONG: &str = "ping_pong_v1";
84const CAP_SESSION_ATTACH: &str = "session_attach_v1";
85const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
86
87const SUBC_CONTROL_OPS: &[&str] = &[
88 ops::ACTIVITY_RECORD,
89 ops::ACTIVITY_QUERY,
90 ops::ACTIVITY_SUBSCRIBE,
91 ops::SERVER_DESCRIBE,
92 ops::CATALOG_LIST,
93 ops::ROUTE_OPEN,
94 ops::ROUTE_POLL,
95 ops::ROUTE_CLOSING,
96 ops::ROUTE_CLOSED,
97 ops::SUPERVISOR_LIST,
98 ops::SUPERVISOR_RESTART,
99 ops::SUPERVISOR_SWAP,
100 ops::SUPERVISOR_RELOAD,
101 ops::SUPERVISOR_RESCAN,
102 ops::SUPERVISOR_RELEASE_RESERVED,
103 ops::SUPERVISOR_SET_ENABLED,
104 ops::SUPERVISOR_HEALTH_PROBE,
105 ops::SUPERVISOR_HEALTH,
106 ops::SUPERVISOR_STDERR_TAIL,
107 ops::SUPERVISOR_TERMINALS,
108 ops::SUPERVISOR_ROUTES,
109 ops::SUPERVISOR_PROVENANCE,
110 ops::SUPERVISOR_SPAWN_SNAPSHOT,
111 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
112];
113
114const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
115 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
116 "supervisor.live_roots",
117 SCOPE_SYNC_OP,
118 SCOPE_APPLY_OP,
119 SCOPE_DESCRIBE_OP,
120 "operator.confirm",
121];
122
123const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
127
128const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
129
130pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
138
139pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
154
155pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
173
174const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
175const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
176
177fn reload_verdict(
178 configured: &Path,
179 spawned_from: Option<&Path>,
180 image: subc_control::RunningImageAgreement,
181) -> PendingReloadVerdict {
182 let path = match spawned_from {
183 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
184 ReloadPathAgreement::Match
185 }
186 Some(spawned_from) => ReloadPathAgreement::Mismatch {
187 configured: configured.to_path_buf(),
188 spawned_from: spawned_from.to_path_buf(),
189 },
190 None => ReloadPathAgreement::Unavailable {
191 reason: if matches!(
192 image,
193 subc_control::RunningImageAgreement::Unavailable {
194 reason: subc_control::RunningImageUnavailableReason::NotRunning
195 }
196 ) {
197 ReloadPathUnavailableReason::NotRunning
198 } else {
199 ReloadPathUnavailableReason::SpawnedPathUnavailable
200 },
201 },
202 };
203 PendingReloadVerdict { path, image }
204}
205
206fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
207 if configured == spawned_from {
208 return true;
209 }
210 #[cfg(windows)]
211 {
212 match (
218 std::fs::canonicalize(configured),
219 std::fs::canonicalize(spawned_from),
220 ) {
221 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
222 _ => false,
223 }
224 }
225 #[cfg(not(windows))]
226 {
227 false
228 }
229}
230
231#[derive(Clone)]
232struct DaemonProvenanceFacts {
233 build: DaemonBuildProvenance,
234 pid: Option<u32>,
235 started_at_ms: Option<u64>,
236 start_clock: Option<crate::clock::StartClock>,
237 executable_path: Option<PathBuf>,
238 executable_identity: Option<SpawnedFileIdentity>,
239 process_start_time: Option<u64>,
240 probe: ExecutableIdentityProbe,
241}
242
243impl Default for DaemonProvenanceFacts {
244 fn default() -> Self {
245 Self {
246 build: DaemonBuildProvenance {
247 build_git_sha: None,
248 build_lock_digest: None,
249 },
250 pid: None,
251 started_at_ms: None,
252 start_clock: None,
253 executable_path: None,
254 executable_identity: None,
255 process_start_time: None,
256 probe: ExecutableIdentityProbe::default(),
257 }
258 }
259}
260
261#[derive(Debug, Clone)]
262struct SupervisorRescanContext {
263 supervisor: Supervisor,
264 config_path: PathBuf,
265 configured_port: Option<u16>,
266 storage_config: Option<crate::daemon_config::StorageConfig>,
267 admission_facts_carrier_module_id: Option<String>,
268 admission_facts_targets: Option<Vec<String>>,
269 scope_authority_owners: Vec<String>,
270}
271
272const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
293 "reloading",
294 "supervisor_not_live",
295 "registration_not_active",
296 "no_forwarding_connection",
297 "relay_send_failed",
298];
299
300#[derive(Clone)]
302pub struct ControlHandler {
303 registry: Arc<Registry>,
304 activity: Option<crate::activity::ActivityLedger>,
305 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
306 forwarding: Arc<ForwardingTable>,
307 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
308 supervisor: SupervisorHandle,
309 subc_capabilities: Arc<[String]>,
310 route_bind_relay_timeout: Duration,
314 route_bind_relay_timeouts: BTreeMap<String, Duration>,
318 route_bind_breakers: RouteBindBreakers,
321 route_bind_concurrency: RouteBindConcurrency,
324 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
329 route_bind_breaker_threshold: u32,
331 route_bind_breaker_cooldown: Duration,
333 health_probe_timeout: Duration,
334 storage_config: Option<crate::daemon_config::StorageConfig>,
337 machine_id: Option<crate::machine_id::MachineId>,
341 admission_facts_carrier_module_id: Option<String>,
342 admission_facts_targets: Option<Vec<String>>,
343 scopes: Arc<RwLock<ScopeTable>>,
347 scope_deadline_added: Arc<tokio::sync::Notify>,
351 scope_authority_owners: Vec<String>,
354 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
357 rescan: Option<SupervisorRescanContext>,
358 connected_clients: ConnectedClients,
359 counters: DaemonCounters,
360 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
361 daemon_provenance: DaemonProvenanceFacts,
362 #[cfg(test)]
363 control_dispatch_delay: Option<Duration>,
364 #[cfg(test)]
365 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
366 #[cfg(test)]
370 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
371}
372
373impl fmt::Debug for ControlHandler {
374 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
375 f.debug_struct("ControlHandler")
376 .field("registry", &self.registry)
377 .field("forwarding", &self.forwarding)
378 .field("process_liveness", &self.process_liveness.is_some())
379 .field("supervisor", &self.supervisor)
380 .field("subc_capabilities", &self.subc_capabilities)
381 .finish()
382 }
383}
384
385struct RouteOpenRequest {
386 target: RouteTarget,
387 identity: BindIdentity,
388 consumer_identity: Option<ConsumerIdentity>,
389 consumer_capabilities: Option<Vec<String>>,
390 role_versions: Option<BTreeMap<String, String>>,
391 admission_facts: Option<serde_json::Value>,
392 scope: Option<ScopeSelector>,
393}
394
395struct RouteBindReservationGuard {
396 forwarding: Arc<ForwardingTable>,
397 endpoint: ModuleEndpointId,
398 relay_corr: u64,
399 armed: bool,
400}
401
402struct ModuleControlRpcGuard {
403 forwarding: Arc<ForwardingTable>,
404 endpoint: ModuleEndpointId,
405 corr: u64,
406 armed: bool,
407}
408
409impl ModuleControlRpcGuard {
410 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
411 Self {
412 forwarding,
413 endpoint,
414 corr,
415 armed: true,
416 }
417 }
418
419 fn disarm(&mut self) {
420 self.armed = false;
421 }
422}
423
424impl Drop for ModuleControlRpcGuard {
425 fn drop(&mut self) {
426 if self.armed {
427 let _ = self
428 .forwarding
429 .cancel_module_control_rpc(self.endpoint, self.corr);
430 }
431 }
432}
433
434impl RouteBindReservationGuard {
435 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
436 Self {
437 forwarding,
438 endpoint,
439 relay_corr,
440 armed: true,
441 }
442 }
443
444 fn release_and_disarm(&mut self) {
445 if !self.armed {
446 return;
447 }
448 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
449 self.endpoint,
450 self.relay_corr,
451 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
452 ) {
453 send_goodbye_target_best_effort(
454 &self.forwarding.counters(),
455 &target,
456 "canceled route.bind",
457 );
458 }
459 self.armed = false;
460 }
461
462 fn disarm(&mut self) {
463 self.armed = false;
464 }
465}
466
467impl Drop for RouteBindReservationGuard {
468 fn drop(&mut self) {
469 self.release_and_disarm();
470 }
471}
472
473#[derive(Debug, Clone, Default)]
503pub(crate) struct RouteBindBreakers {
504 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
505}
506
507#[derive(Debug, Clone, Default)]
508pub(crate) struct RouteBindConcurrency {
509 modules: Arc<Mutex<HashMap<String, usize>>>,
510}
511
512struct RouteBindConcurrencyGuard {
513 concurrency: RouteBindConcurrency,
514 module_id: String,
515}
516
517impl RouteBindConcurrency {
518 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
521 let mut modules = self
522 .modules
523 .lock()
524 .expect("route.bind concurrency mutex poisoned");
525 let in_flight = modules.entry(module_id.to_string()).or_default();
526 if *in_flight >= limit {
527 return Err(*in_flight);
528 }
529 *in_flight += 1;
530 Ok(RouteBindConcurrencyGuard {
531 concurrency: self.clone(),
532 module_id: module_id.to_string(),
533 })
534 }
535}
536
537impl Drop for RouteBindConcurrencyGuard {
538 fn drop(&mut self) {
539 let mut modules = self
540 .concurrency
541 .modules
542 .lock()
543 .expect("route.bind concurrency mutex poisoned");
544 let remove = {
545 let in_flight = modules
546 .get_mut(&self.module_id)
547 .expect("admitted route.bind has a concurrency entry");
548 *in_flight -= 1;
549 *in_flight == 0
550 };
551 if remove {
552 modules.remove(&self.module_id);
553 }
554 }
555}
556
557#[derive(Debug, Default)]
558struct ModuleBreakerState {
559 consecutive_timeouts: u32,
561 cooldown_until: Option<Instant>,
564 probe_in_flight: Option<Arc<()>>,
569}
570
571enum RouteBindAdmission<'a> {
573 Admitted {
574 guard: RouteBindBreakerGuard<'a>,
575 probe: bool,
578 },
579 Refused {
580 consecutive_timeouts: u32,
581 retry_in: Duration,
585 probe_in_flight: bool,
586 },
587}
588
589struct RouteBindBreakerGuard<'a> {
596 breakers: RouteBindBreakers,
597 module_id: &'a str,
598 probe_token: Option<Arc<()>>,
599 settled: bool,
600}
601
602impl RouteBindBreakerGuard<'_> {
603 fn record_accepted(&mut self) -> bool {
607 self.settled = true;
608 self.breakers.record_accepted(self.module_id)
609 }
610
611 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
614 self.settled = true;
615 self.breakers.record_timeout(
616 self.module_id,
617 self.probe_token.as_ref(),
618 threshold,
619 cooldown,
620 )
621 }
622
623 fn record_inconclusive(&mut self) {
632 self.settled = true;
633 self.breakers
634 .record_inconclusive(self.module_id, self.probe_token.as_ref());
635 }
636}
637
638impl Drop for RouteBindBreakerGuard<'_> {
639 fn drop(&mut self) {
640 if !self.settled {
641 self.breakers
642 .record_inconclusive(self.module_id, self.probe_token.as_ref());
643 }
644 }
645}
646
647struct BreakerOpened {
651 consecutive_timeouts: u32,
652 reopened_after_probe: bool,
655}
656
657impl RouteBindBreakers {
658 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
659 self.modules
660 .lock()
661 .expect("route.bind breaker mutex poisoned")
662 }
663
664 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
667 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
668 probe: probe_token.is_some(),
669 guard: RouteBindBreakerGuard {
670 breakers: self.clone(),
671 module_id,
672 probe_token,
673 settled: false,
674 },
675 };
676
677 let mut modules = self.lock();
678 let Some(state) = modules.get_mut(module_id) else {
679 return admitted(None);
680 };
681 let Some(cooldown_until) = state.cooldown_until else {
682 return admitted(None);
683 };
684 if state.probe_in_flight.is_some() {
685 return RouteBindAdmission::Refused {
686 consecutive_timeouts: state.consecutive_timeouts,
687 retry_in: Duration::ZERO,
688 probe_in_flight: true,
689 };
690 }
691 let now = Instant::now();
692 if now < cooldown_until {
693 return RouteBindAdmission::Refused {
694 consecutive_timeouts: state.consecutive_timeouts,
695 retry_in: cooldown_until - now,
696 probe_in_flight: false,
697 };
698 }
699 let token = Arc::new(());
700 state.probe_in_flight = Some(Arc::clone(&token));
701 admitted(Some(token))
702 }
703
704 fn record_accepted(&self, module_id: &str) -> bool {
705 self.lock()
706 .remove(module_id)
707 .is_some_and(|state| state.cooldown_until.is_some())
708 }
709
710 fn record_timeout(
711 &self,
712 module_id: &str,
713 probe_token: Option<&Arc<()>>,
714 threshold: u32,
715 cooldown: Duration,
716 ) -> Option<BreakerOpened> {
717 let mut modules = self.lock();
718 let state = modules.entry(module_id.to_string()).or_default();
719 let was_open = state.cooldown_until.is_some();
720 let was_probe = Self::owns_probe(state, probe_token);
721 if was_probe {
722 state.probe_in_flight = None;
723 }
724 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
725 if state.consecutive_timeouts < threshold {
726 return None;
727 }
728 state.cooldown_until = Some(Instant::now() + cooldown);
729 Some(BreakerOpened {
730 consecutive_timeouts: state.consecutive_timeouts,
731 reopened_after_probe: was_open && was_probe,
732 })
733 }
734
735 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
736 state
740 .probe_in_flight
741 .as_ref()
742 .zip(token)
743 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
744 }
745
746 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
747 if let Some(state) = self.lock().get_mut(module_id) {
748 if Self::owns_probe(state, probe_token) {
749 state.probe_in_flight = None;
750 }
751 }
752 }
753
754 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
769 self.lock()
770 .remove(module_id)
771 .map(|state| state.consecutive_timeouts)
772 .filter(|discarded| *discarded > 0)
773 }
774
775 fn open_snapshot(&self) -> Option<serde_json::Value> {
782 let now = Instant::now();
783 let modules = self.lock();
784 let open = modules
785 .iter()
786 .filter_map(|(module_id, state)| {
787 let cooldown_until = state.cooldown_until?;
788 Some((
789 module_id.clone(),
790 serde_json::json!({
791 "consecutive_timeouts": state.consecutive_timeouts,
792 "cooldown_remaining_ms":
793 cooldown_until.saturating_duration_since(now).as_millis() as u64,
794 "probe_in_flight": state.probe_in_flight.is_some(),
795 }),
796 ))
797 })
798 .collect::<serde_json::Map<String, serde_json::Value>>();
799 (!open.is_empty()).then_some(serde_json::Value::Object(open))
800 }
801}
802
803impl ControlHandler {
804 pub fn new(registry: Arc<Registry>) -> Self {
805 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
806 }
807
808 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
809 let counters = forwarding.counters();
810 let route_bind_breakers = forwarding.route_bind_breakers();
814 let route_bind_concurrency = forwarding.route_bind_concurrency();
815 let route_outages = forwarding.route_outages();
816 Self {
817 registry,
818 activity: None,
819 activity_streams: Arc::new(Mutex::new(HashMap::new())),
820 forwarding,
821 process_liveness: None,
822 supervisor: SupervisorHandle::new(),
823 subc_capabilities: Arc::from([
824 CAP_MANIFEST_REGISTRATION.to_string(),
825 CAP_CHANNEL_LIFECYCLE.to_string(),
826 CAP_PING_PONG.to_string(),
827 CAP_SESSION_ATTACH.to_string(),
828 CAP_ADMISSION_FACTS_RELAY.to_string(),
829 CAP_SCOPES_V1.to_string(),
830 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
831 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
832 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
833 ]),
834 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
835 route_bind_relay_timeouts: BTreeMap::new(),
836 route_bind_breakers,
837 route_bind_concurrency,
838 route_outages,
839 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
840 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
841 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
842 storage_config: None,
843 machine_id: None,
844 admission_facts_carrier_module_id: None,
845 admission_facts_targets: None,
846 scopes: Arc::new(RwLock::new(ScopeTable::new(
847 crate::daemon_config::default_scope_authority_owners(),
848 ))),
849 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
850 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
851 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
852 rescan: None,
853 connected_clients: ConnectedClients::new(),
854 counters,
855 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
856 daemon_provenance: DaemonProvenanceFacts::default(),
857 #[cfg(test)]
858 control_dispatch_delay: None,
859 #[cfg(test)]
860 provenance_probe_override: None,
861 #[cfg(test)]
862 route_bind_relay_sent_hold: None,
863 }
864 }
865
866 pub fn with_storage_config(
869 mut self,
870 storage_config: Option<crate::daemon_config::StorageConfig>,
871 ) -> Self {
872 self.storage_config = storage_config;
873 self
874 }
875
876 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
879 self.activity = Some(crate::activity::ActivityLedger::open(
880 run_dir.join("activity.jsonl"),
881 incarnation,
882 ));
883 self
884 }
885
886 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
889 self.machine_id = machine_id;
890 self
891 }
892
893 pub fn with_admission_facts_config(
897 mut self,
898 carrier_module_id: Option<String>,
899 targets: Option<Vec<String>>,
900 ) -> Self {
901 self.admission_facts_carrier_module_id = carrier_module_id;
902 self.admission_facts_targets = targets;
903 self
904 }
905
906 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
910 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
911 self.scope_authority_owners = owners;
912 self
913 }
914
915 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
918 self.route_bind_relay_timeout = timeout;
919 self
920 }
921
922 pub fn with_route_bind_relay_timeouts(
928 mut self,
929 timeouts: impl IntoIterator<Item = (String, Duration)>,
930 ) -> Self {
931 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
932 self
933 }
934
935 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
941 self.route_bind_relay_timeouts
942 .get(module_id)
943 .copied()
944 .unwrap_or(self.route_bind_relay_timeout)
945 }
946
947 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
955 self.route_bind_breaker_threshold = threshold.max(1);
956 self.route_bind_breaker_cooldown = cooldown;
957 self
958 }
959
960 #[cfg(test)]
961 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
962 self.health_probe_timeout = timeout;
963 self
964 }
965
966 #[cfg(test)]
967 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
968 self.control_dispatch_delay = Some(delay);
969 self
970 }
971
972 #[cfg(test)]
976 pub(crate) fn with_route_bind_relay_sent_hold(
977 mut self,
978 hold: Arc<tokio::sync::Barrier>,
979 ) -> Self {
980 self.route_bind_relay_sent_hold = Some(hold);
981 self
982 }
983
984 pub fn with_process_liveness(
985 mut self,
986 process_liveness: Arc<dyn ModuleProcessLiveness>,
987 ) -> Self {
988 self.process_liveness = Some(process_liveness);
989 self
990 }
991
992 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
993 self.supervisor = supervisor;
994 self
995 }
996
997 pub fn with_daemon_provenance(
998 mut self,
999 pid: u32,
1000 started_at_ms: u64,
1001 executable_path: Option<PathBuf>,
1002 build_git_sha: Option<String>,
1003 build_lock_digest: Option<String>,
1004 ) -> Self {
1005 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1006 let process_start_time = process_start_time(pid);
1007 self.daemon_provenance = DaemonProvenanceFacts {
1008 build: DaemonBuildProvenance {
1009 build_git_sha,
1010 build_lock_digest,
1011 },
1012 pid: Some(pid),
1013 started_at_ms: Some(started_at_ms),
1014 start_clock: None,
1015 executable_path,
1016 executable_identity,
1017 process_start_time,
1018 probe: ExecutableIdentityProbe::default(),
1019 };
1020 self
1021 }
1022
1023 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1024 self.daemon_provenance.start_clock = Some(clock);
1025 self
1026 }
1027
1028 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1030 self.scopes
1031 .write()
1032 .unwrap_or_else(|poisoned| poisoned.into_inner())
1033 .set_wall_clock(Arc::new(clock));
1034 self
1035 }
1036
1037 #[cfg(test)]
1038 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1039 self.provenance_probe_override = Some(result);
1040 self
1041 }
1042
1043 pub fn with_capability_config(
1047 self,
1048 modules: impl IntoIterator<Item = (String, bool)>,
1049 reserved_capabilities: BTreeMap<String, String>,
1050 ) -> Self {
1051 self.capability_evaluator
1052 .configure(modules, reserved_capabilities);
1053 self
1054 }
1055
1056 pub fn with_supervisor_rescan(
1057 mut self,
1058 supervisor: Supervisor,
1059 config_path: impl Into<PathBuf>,
1060 configured_port: Option<u16>,
1061 ) -> Self {
1062 self.rescan = Some(SupervisorRescanContext {
1063 supervisor,
1064 config_path: config_path.into(),
1065 configured_port,
1066 storage_config: self.storage_config.clone(),
1067 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1068 admission_facts_targets: self.admission_facts_targets.clone(),
1069 scope_authority_owners: self.scope_authority_owners.clone(),
1070 });
1071 self
1072 }
1073
1074 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1075 self.connected_clients = connected_clients;
1076 self
1077 }
1078
1079 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1080 Arc::clone(&self.forwarding)
1081 }
1082
1083 pub(crate) fn counters(&self) -> DaemonCounters {
1084 self.counters.clone()
1085 }
1086
1087 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1090 tokio::spawn(async move {
1091 loop {
1092 self.capability_evaluator
1093 .wait_for_change_or_deadline()
1094 .await;
1095 self.refresh_capability_requirements();
1096 }
1097 });
1098 }
1099
1100 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1106 tokio::spawn(async move {
1107 loop {
1108 let has_deadlines = self
1109 .scopes
1110 .read()
1111 .unwrap_or_else(|poisoned| poisoned.into_inner())
1112 .has_deadlines();
1113 if !has_deadlines {
1114 self.scope_deadline_added.notified().await;
1115 continue;
1116 }
1117 tokio::time::sleep(Duration::from_secs(1)).await;
1118 if let Err(error) = self.sweep_expired_scopes() {
1119 warn!(%error, "scope expiry sweep failed");
1120 }
1121 }
1122 });
1123 }
1124
1125 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1126 let mut table = self
1127 .scopes
1128 .write()
1129 .unwrap_or_else(|poisoned| poisoned.into_inner());
1130 let swept = table.sweep_expired();
1131 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1132 drop(table);
1133 self.close_scope_drained_routes(drained);
1134 Ok(())
1135 }
1136
1137 fn publish_scope_changes(
1138 &self,
1139 changes: &[crate::scopes::ScopeTagChange],
1140 expired: &[crate::scopes::ScopeExpired],
1141 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1142 if changes.is_empty() {
1143 return Ok(Vec::new());
1144 }
1145 let drained = self
1146 .forwarding
1147 .publish_scope_changes(changes)
1148 .map_err(RouterError::Forwarding)?;
1149 let mut counts = HashMap::new();
1150 for route in &drained {
1151 *counts
1152 .entry((
1153 route.scope.owner.as_str(),
1154 route.scope.scope_ref.as_str(),
1155 route.scope.tag.scope_epoch,
1156 ))
1157 .or_insert(0usize) += 1;
1158 }
1159 for scope in expired {
1160 let routes_closed = counts
1161 .get(&(
1162 scope.owner.as_str(),
1163 scope.scope_ref.as_str(),
1164 scope.scope_epoch,
1165 ))
1166 .copied()
1167 .unwrap_or(0);
1168 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1169 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1170 routes_closed, "scope expired");
1171 }
1172 Ok(drained)
1173 }
1174
1175 fn runtime_capability_snapshot(
1176 &self,
1177 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1178 let runtime = self
1179 .supervisor
1180 .list()
1181 .into_iter()
1182 .map(|module| {
1183 let status = module.status().map_err(|err| {
1184 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1185 })?;
1186 Ok(RuntimeModule {
1187 module_id: status.module_id,
1188 state: status.state,
1189 enabled: status.enabled,
1190 })
1191 })
1192 .collect::<Result<Vec<_>, RouterError>>()?;
1193 let (_, registrations) = self.registry.list_modules().map_err(|err| {
1194 RouterError::backend(
1195 0,
1196 0,
1197 format!("failed to list capability registrations: {err}"),
1198 )
1199 })?;
1200 let registrations = registrations
1201 .into_iter()
1202 .map(|registration| RegisteredModule {
1203 module_id: registration.manifest.module_id,
1204 module_version: registration.manifest.module_version,
1205 capabilities: registration.manifest.capabilities,
1206 })
1207 .collect();
1208 Ok((runtime, registrations))
1209 }
1210
1211 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1218 let cached_registration = RegisteredModule {
1219 module_id: registration.manifest.module_id.clone(),
1220 module_version: registration.manifest.module_version.clone(),
1221 capabilities: registration.manifest.capabilities.clone(),
1222 };
1223 if self.capability_evaluator.record_hello(&cached_registration) {
1224 warn!(
1225 module_id = %cached_registration.module_id,
1226 "capability claims drifted from the cached manifest"
1227 );
1228 }
1229 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1230 self.enforce_capability_denies();
1231 }
1232 self.refresh_capability_requirements();
1233 }
1234
1235 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1239 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1240 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1241 self.supervisor.set_swap_promotion_observer(observer);
1242 }
1243
1244 pub fn refresh_capability_requirements(&self) {
1245 match self.runtime_capability_snapshot() {
1246 Ok((runtime, registrations)) => {
1247 log_requirement_events(
1248 self.capability_evaluator
1249 .evaluate_now(&runtime, ®istrations),
1250 );
1251 }
1252 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1253 }
1254 }
1255
1256 fn enforce_capability_denies(&self) {
1260 let (_, registrations) = match self.registry.list_modules() {
1261 Ok(snapshot) => snapshot,
1262 Err(err) => {
1263 warn!(error = %err, "failed to read registrations for capability deny census");
1264 return;
1265 }
1266 };
1267 let manifests = registrations
1268 .into_iter()
1269 .map(|registration| {
1270 (
1271 registration.manifest.module_id.clone(),
1272 registration.manifest,
1273 )
1274 })
1275 .collect::<BTreeMap<_, _>>();
1276 let census = match self.forwarding.route_census(None) {
1277 Ok(census) => census,
1278 Err(err) => {
1279 warn!(error = %err, "failed to read route census for capability deny enforcement");
1280 return;
1281 }
1282 };
1283
1284 for (target_module_id, routes) in census {
1285 let Some(target_manifest) = manifests.get(&target_module_id) else {
1286 continue;
1287 };
1288 let mut closed_routes = Vec::new();
1289 let mut module_goodbyes = Vec::new();
1290 for route in routes {
1291 let Principal::Reserved {
1292 module_id: opening_module_id,
1293 } = &route.principal
1294 else {
1295 continue;
1296 };
1297 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1298 continue;
1299 };
1300 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1301 continue;
1302 };
1303
1304 match self.forwarding.release_client_route(
1305 route.goodbye_target.connection_id,
1306 route.goodbye_target.channel,
1307 route.goodbye_target.epoch,
1308 ) {
1309 Ok(RouteRelease::Removed(module_goodbye)) => {
1310 warn!(
1311 opening_module_id,
1312 target_module_id,
1313 capability,
1314 "force-closing route because an attested capability deny edge now matches"
1315 );
1316 closed_routes.push(route);
1317 module_goodbyes.push(module_goodbye);
1318 }
1319 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1320 Err(err) => warn!(
1321 opening_module_id,
1322 target_module_id,
1323 capability,
1324 error = %err,
1325 "failed to force-close capability-denied route"
1326 ),
1327 }
1328 }
1329
1330 if closed_routes.is_empty() {
1331 continue;
1332 }
1333 send_route_control_pushes(
1334 &self.forwarding,
1335 closed_routes,
1336 ClientControlPush::RouteClosed {
1337 module_id: target_module_id,
1338 channels: Vec::new(),
1339 reason: RouteCloseReason::CapabilityDenied,
1340 drained: false,
1341 abandoned: 0,
1342 excluded_subscriptions: 0,
1343 terminal: Some(false),
1344 },
1345 );
1346 self.emit_route_goodbyes(module_goodbyes);
1347 }
1348 }
1349
1350 fn not_ready_reason(
1356 &self,
1357 registration: &crate::registry::ModuleRegistration,
1358 ) -> Option<NotReadyReason> {
1359 if !registration.ready {
1360 return Some(NotReadyReason {
1361 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1362 capability: None,
1363 });
1364 }
1365 self.first_unprovided_required_capability(registration)
1366 .map(|capability| NotReadyReason {
1367 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1368 capability: Some(capability),
1369 })
1370 }
1371
1372 fn first_unprovided_required_capability(
1390 &self,
1391 registration: &crate::registry::ModuleRegistration,
1392 ) -> Option<String> {
1393 let required = registration
1394 .manifest
1395 .capabilities
1396 .iter()
1397 .flat_map(|declarations| declarations.requires.iter())
1398 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1399 .map(|requirement| requirement.capability.as_str())
1400 .collect::<BTreeSet<_>>();
1401 if required.is_empty() {
1402 return None;
1403 }
1404 let module_id = registration.manifest.module_id.as_str();
1405 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1406 if required
1407 .iter()
1408 .any(|capability| verdict(capability).is_none())
1409 {
1410 self.refresh_capability_requirements();
1411 }
1412 required
1413 .into_iter()
1414 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1415 .map(str::to_string)
1416 }
1417
1418 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1419 self.capability_evaluator
1420 .statuses()
1421 .into_iter()
1422 .map(capability_requirement_status)
1423 .collect()
1424 }
1425
1426 fn deregister_connection(
1433 &self,
1434 connection_id: ConnectionId,
1435 reason: RegistrationEndReason,
1436 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1437 self.registry
1438 .deregister_connection_with_reason(connection_id, reason)
1439 }
1440
1441 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1442 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1443 return None;
1444 }
1445 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1446 parse_client_control_request(&frame.body)
1447 else {
1448 return None;
1449 };
1450 Some(target_module_id(&target).to_string())
1451 }
1452
1453 pub(crate) fn route_open_capacity_refusal(
1454 &self,
1455 ctx: &RouteCtx,
1456 frame: &Frame,
1457 target_module_id: &str,
1458 in_flight: usize,
1459 limit: usize,
1460 ) -> Result<Frame, RouterError> {
1461 self.route_open_admission_refusal_frame(
1462 ctx,
1463 frame,
1464 target_module_id,
1465 "open_admission_full",
1466 (in_flight, limit),
1467 format!(
1468 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1469 ),
1470 )
1471 }
1472
1473 fn route_open_target_capacity_refusal(
1474 &self,
1475 ctx: &RouteCtx,
1476 frame: &Frame,
1477 target_module_id: &str,
1478 in_flight: usize,
1479 ) -> Result<Frame, RouterError> {
1480 self.route_open_admission_refusal_frame(
1481 ctx,
1482 frame,
1483 target_module_id,
1484 "target_binds_full",
1485 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1486 format!(
1487 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1488 ),
1489 )
1490 }
1491
1492 fn route_open_admission_refusal_frame(
1500 &self,
1501 ctx: &RouteCtx,
1502 frame: &Frame,
1503 target_module_id: &str,
1504 reason: &'static str,
1505 (in_flight, limit): (usize, usize),
1506 message: impl Into<String>,
1507 ) -> Result<Frame, RouterError> {
1508 let code = error_codes::TARGET_UNAVAILABLE;
1509 self.counters.increment_route_open_refused(code);
1510 info!(
1511 target: "control",
1512 code,
1513 reason,
1514 module_id = ?target_module_id,
1515 connection_id = ctx.connection_id.get(),
1516 in_flight,
1517 limit,
1518 "route.open refused"
1519 );
1520 control_error_frame(frame, code, message.into())
1521 }
1522
1523 #[cfg(test)]
1529 pub fn handle_control(
1530 &self,
1531 connection_id: ConnectionId,
1532 frame: Frame,
1533 ) -> Result<Vec<Frame>, RouterError> {
1534 match frame.header.ty {
1535 FrameType::Ping => Ok(vec![pong(&frame)?]),
1536 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1537 FrameType::Goodbye => self.handle_goodbye(connection_id),
1538 ty => Ok(vec![control_error_frame(
1539 &frame,
1540 "unsupported_control_frame",
1541 format!("unsupported channel-0 frame {ty:?}"),
1542 )?]),
1543 }
1544 }
1545
1546 pub async fn handle_control_frame(
1547 &self,
1548 ctx: &RouteCtx,
1549 frame: Frame,
1550 ) -> Result<Vec<Frame>, RouterError> {
1551 self.handle_control_frame_timed(ctx, frame, None).await
1552 }
1553
1554 pub(crate) async fn handle_control_frame_timed(
1555 &self,
1556 ctx: &RouteCtx,
1557 frame: Frame,
1558 dispatch_started_at: Option<StdInstant>,
1559 ) -> Result<Vec<Frame>, RouterError> {
1560 match frame.header.ty {
1561 FrameType::Ping => Ok(vec![pong(&frame)?]),
1562 FrameType::Hello => {
1563 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1564 }
1565 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1566 FrameType::Cancel => {
1567 if self
1570 .forwarding
1571 .operator_confirms()
1572 .cancel(ctx.connection_id, frame.header.corr)
1573 || self
1574 .supervisor
1575 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1576 || self.cancel_activity_subscription(ctx, &frame)?
1577 {
1578 Ok(Vec::new())
1579 } else {
1580 Ok(vec![control_error_frame(
1581 &frame,
1582 "unknown_subscription",
1583 "no subscription has this correlation id",
1584 )?])
1585 }
1586 }
1587 FrameType::Request => {
1588 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1591 if op
1592 .as_ref()
1593 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1594 {
1595 return self.handle_activity_record(ctx, frame).await;
1596 }
1597 if op.as_ref().is_some_and(|probe| {
1598 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1599 }) {
1600 return self.handle_activity_read(ctx, frame).await;
1601 }
1602 if op
1603 .as_ref()
1604 .is_some_and(|probe| probe.op == "operator.confirm")
1605 {
1606 return self.handle_operator_confirm(ctx, frame);
1607 }
1608 if self
1609 .forwarding
1610 .module_endpoint_for_connection(ctx.connection_id)
1611 .map_err(RouterError::Forwarding)?
1612 .is_some()
1613 {
1614 if !is_known_module_request_op(&frame.body) {
1615 return Ok(vec![control_error_frame(
1616 &frame,
1617 "unsupported_control_frame",
1618 "module-originated channel-0 REQUEST is not supported",
1619 )?]);
1620 }
1621 let request = match parse_module_control_request_from_module(&frame.body) {
1622 Ok(request) => request,
1623 Err((err, ControlRequestBodyError::UnknownOp)) => {
1624 return Ok(vec![control_error_frame(
1625 &frame,
1626 "unsupported_control_frame",
1627 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1628 )?])
1629 }
1630 Err((err, ControlRequestBodyError::InvalidBody)) => {
1631 return Ok(vec![control_error_frame(
1632 &frame,
1633 "invalid_control_body",
1634 format!("malformed module control body: {err}"),
1635 )?])
1636 }
1637 };
1638 let op = module_control_request_op(&request);
1639 let corr = frame.header.corr;
1640 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1641 let result =
1642 self.handle_module_control_request(ctx.connection_id, frame, request);
1643 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1644 return result;
1645 }
1646
1647 if is_known_module_request_op(&frame.body) {
1648 return Ok(vec![control_error_frame(
1649 &frame,
1650 "not_registered",
1651 "catalog.update requires an active module registration owned by this connection",
1652 )?]);
1653 }
1654
1655 let request = match parse_client_control_request(&frame.body) {
1656 Ok(request) => request,
1657 Err((err, ControlRequestBodyError::UnknownOp)) => {
1658 return Ok(vec![control_error_frame(
1659 &frame,
1660 "unknown_control_op",
1661 format!("unknown client control op: {err}"),
1662 )?])
1663 }
1664 Err((err, ControlRequestBodyError::InvalidBody)) => {
1665 return Ok(vec![control_error_frame(
1666 &frame,
1667 "invalid_control_body",
1668 format!("malformed client control body: {err}"),
1669 )?])
1670 }
1671 };
1672 let op = client_control_request_op(&request);
1673 let corr = frame.header.corr;
1674 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1675 #[cfg(test)]
1676 if let Some(delay) = self.control_dispatch_delay {
1677 tokio::time::sleep(delay).await;
1678 }
1679 let result = self
1680 .handle_client_control_request(ctx, frame, request)
1681 .await;
1682 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1683 result
1684 }
1685 FrameType::Push => {
1686 let Some(endpoint) = self
1687 .forwarding
1688 .module_endpoint_for_connection(ctx.connection_id)
1689 .map_err(RouterError::Forwarding)?
1690 else {
1691 return Ok(vec![control_error_frame(
1692 &frame,
1693 "unsupported_control_frame",
1694 "client-originated channel-0 PUSH is not supported",
1695 )?]);
1696 };
1697 self.handle_status_update(endpoint, frame)
1698 }
1699 FrameType::Response | FrameType::Error
1700 if self
1701 .forwarding
1702 .module_endpoint_for_connection(ctx.connection_id)
1703 .map_err(RouterError::Forwarding)?
1704 .is_some() =>
1705 {
1706 self.handle_module_relay_response(ctx.connection_id, frame)
1707 }
1708 ty => Ok(vec![control_error_frame(
1709 &frame,
1710 "unsupported_control_frame",
1711 format!("unsupported channel-0 frame {ty:?}"),
1712 )?]),
1713 }
1714 }
1715
1716 pub fn cleanup_connection(
1717 &self,
1718 connection_id: ConnectionId,
1719 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1720 self.cleanup_connection_with_end_reason(
1721 connection_id,
1722 RegistrationEndReason::ConnectionClosed,
1723 )
1724 }
1725
1726 fn cleanup_connection_with_end_reason(
1727 &self,
1728 connection_id: ConnectionId,
1729 requested_reason: RegistrationEndReason,
1730 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1731 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1732 self.registry
1733 .get_module_by_connection(connection_id)?
1734 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1735 .and_then(|module| module.registration_end_reason().ok().flatten())
1736 .unwrap_or(requested_reason)
1737 } else {
1738 requested_reason
1739 };
1740 let crash_closed = self
1741 .registry
1742 .get_module_by_connection(connection_id)?
1743 .and_then(|registration| {
1744 self.forwarding
1745 .module_endpoint_for_connection(connection_id)
1746 .ok()
1747 .flatten()
1748 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1749 .map(|routes| (registration.manifest.module_id, routes))
1750 });
1751 let crash_closed = crash_closed.map(|(module_id, routes)| {
1752 let terminal = match self.supervisor.get(&module_id) {
1753 None => false,
1754 Some(module) => match module.will_recover_after_connection_loss() {
1755 Ok(will_recover) => !will_recover,
1756 Err(err) => {
1757 warn!(
1758 %module_id,
1759 error = %err,
1760 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1761 );
1762 false
1763 }
1764 },
1765 };
1766 let (reason, drained) = match self
1770 .forwarding
1771 .daemon_drain_result_for_connection(connection_id)
1772 {
1773 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
1774 Ok(None) => (RouteCloseReason::Crash, false),
1775 Err(err) => {
1776 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1777 (RouteCloseReason::Crash, false)
1778 }
1779 };
1780 (module_id, routes, reason, drained, terminal)
1781 });
1782 let registrations = self.deregister_connection(connection_id, end_reason);
1783 let cleanup = if crash_closed.is_some() {
1784 self.forwarding.cleanup_connection_counted(connection_id)
1785 } else {
1786 self.forwarding
1792 .cleanup_connection(connection_id)
1793 .map(|released| crate::forwarding::ConnectionCleanup {
1794 released,
1795 abandoned_relays: 0,
1796 })
1797 };
1798 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
1803 let abandoned = cleanup
1804 .as_ref()
1805 .map_or(0, |cleanup| cleanup.abandoned_relays);
1806 send_route_control_pushes(
1807 &self.forwarding,
1808 routes,
1809 ClientControlPush::RouteClosed {
1810 module_id,
1811 channels: Vec::new(),
1812 reason,
1813 drained,
1814 abandoned,
1815 excluded_subscriptions: 0,
1816 terminal: Some(terminal),
1817 },
1818 );
1819 }
1820 if let Ok(cleanup) = cleanup {
1821 self.emit_route_goodbyes(cleanup.released);
1822 }
1823 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1827 crate::supervise::notify_registration_release();
1828 self.capability_evaluator.wake_deadline_loop();
1829 self.refresh_capability_requirements();
1830 }
1831 self.supervisor.remove_spawn_subscribers(connection_id);
1832 self.activity_streams
1833 .lock()
1834 .unwrap_or_else(|p| p.into_inner())
1835 .retain(|(id, _), task| {
1836 if *id == connection_id {
1837 task.abort();
1838 false
1839 } else {
1840 true
1841 }
1842 });
1843 self.hello_launch_nonces
1846 .lock()
1847 .unwrap_or_else(|poisoned| poisoned.into_inner())
1848 .forget(connection_id);
1849 self.scopes
1850 .write()
1851 .unwrap_or_else(|poisoned| poisoned.into_inner())
1852 .release_connection(connection_id);
1853 registrations
1854 }
1855
1856 pub(crate) fn handle_route_goodbye(
1857 &self,
1858 connection_id: ConnectionId,
1859 route_channel: u16,
1860 route_epoch: u32,
1861 ) -> Result<bool, RouterError> {
1862 debug!(
1863 connection_id = connection_id.get(),
1864 route_channel, route_epoch, "handling route GOODBYE"
1865 );
1866 let RouteRelease::Removed(released_route) = self
1867 .forwarding
1868 .release_client_route(connection_id, route_channel, route_epoch)
1869 .map_err(RouterError::Forwarding)?
1870 else {
1871 return Ok(false);
1872 };
1873 self.emit_route_goodbyes(vec![released_route]);
1874 Ok(true)
1875 }
1876
1877 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1878 for released in released_routes {
1879 let frame = match Frame::build_with_version(
1880 released.negotiated_ver,
1881 FrameType::Goodbye,
1882 control_flags(),
1883 released.channel,
1884 released.epoch,
1885 0,
1886 Vec::new(),
1887 ) {
1888 Ok(frame) => frame,
1889 Err(err) => {
1890 warn!(
1891 route_channel = released.channel,
1892 error = %err,
1893 "failed to build route GOODBYE frame"
1894 );
1895 continue;
1896 }
1897 };
1898 if !released.close_on_delivery_failure() {
1899 crate::forwarding::send_module_route_goodbye(
1900 &self.counters,
1901 &released.sink,
1902 frame,
1903 released.module_id.as_deref(),
1904 "client route released",
1905 );
1906 continue;
1907 }
1908 if let Err(err) = released.sink.try_send(frame) {
1909 warn!(
1910 target_connection_id = released.connection_id.get(),
1911 route_channel = released.channel,
1912 error = %err,
1913 "route GOODBYE was not delivered to client; closing target connection"
1914 );
1915 if self
1916 .forwarding
1917 .escalate_client_delivery_failure(
1918 released.connection_id,
1919 released.channel,
1920 released.epoch,
1921 CloseReason::new(
1922 "route_goodbye_delivery_failed",
1923 format!(
1924 "failed to enqueue route GOODBYE for channel {}: {err}",
1925 released.channel
1926 ),
1927 ),
1928 crate::forwarding::UndeliveredFrame {
1929 module_id: released.module_id.as_deref(),
1930 sink: &released.sink,
1931 },
1932 )
1933 .unwrap_or(false)
1934 {
1935 self.counters.increment_goodbye_relay_client_failed();
1936 }
1937 }
1938 }
1939 }
1940
1941 fn send_abandoned_route_bind_goodbye(
1953 &self,
1954 module_sink: &crate::FrameSink,
1955 negotiated_ver: u8,
1956 module_channel: u16,
1957 module_epoch: u32,
1958 ) {
1959 let frame = match Frame::build_with_version(
1960 negotiated_ver,
1961 FrameType::Goodbye,
1962 control_flags(),
1963 module_channel,
1964 module_epoch,
1965 0,
1966 Vec::new(),
1967 ) {
1968 Ok(frame) => frame,
1969 Err(err) => {
1970 warn!(
1971 route_channel = module_channel,
1972 error = %err,
1973 "failed to build GOODBYE for abandoned route.bind"
1974 );
1975 return;
1976 }
1977 };
1978 crate::forwarding::send_module_route_goodbye(
1979 &self.counters,
1980 module_sink,
1981 frame,
1982 None,
1983 "abandoned route.bind",
1984 );
1985 }
1986
1987 fn handle_hello(
1988 &self,
1989 connection_id: ConnectionId,
1990 sink: Option<crate::FrameSink>,
1991 frame: Frame,
1992 ) -> Result<Vec<Frame>, RouterError> {
1993 debug!(
1994 connection_id = connection_id.get(),
1995 corr = frame.header.corr,
1996 "handling HELLO"
1997 );
1998 if self
2002 .registry
2003 .get_module_by_connection(connection_id)
2004 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2005 .is_some()
2006 {
2007 return Ok(vec![control_error_frame(
2008 &frame,
2009 "invalid_hello",
2010 "connection is already registered as a module",
2011 )?]);
2012 }
2013 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2014 Ok(value) => value,
2015 Err(err) => {
2016 return Ok(vec![control_error_frame(
2017 &frame,
2018 "invalid_hello",
2019 format!("malformed HELLO body: {err}"),
2020 )?])
2021 }
2022 };
2023 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2024 return Ok(vec![control_error_frame(
2025 &frame,
2026 "invalid_capability_grammar",
2027 err.to_string(),
2028 )?]);
2029 }
2030 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2031 return Ok(vec![control_error_frame(
2032 &frame,
2033 "invalid_manifest",
2034 err.to_string(),
2035 )?]);
2036 }
2037 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2038 return Ok(vec![control_error_frame(
2039 &frame,
2040 "invalid_event_declaration",
2041 err.to_string(),
2042 )?]);
2043 }
2044 if let Some(provenance) = hello_value
2045 .get("manifest")
2046 .and_then(|manifest| manifest.get("provenance"))
2047 {
2048 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2049 return Ok(vec![control_error_frame(
2050 &frame,
2051 "invalid_manifest",
2052 format!("malformed manifest provenance: {err}"),
2053 )?]);
2054 }
2055 }
2056 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2057 Ok(hello) => hello,
2058 Err(err) => {
2059 return Ok(vec![control_error_frame(
2060 &frame,
2061 "invalid_hello",
2062 format!("malformed HELLO body: {err}"),
2063 )?])
2064 }
2065 };
2066
2067 if hello.protocol_ver != hello.manifest.protocol_ver {
2068 return Ok(vec![control_error_frame(
2069 &frame,
2070 "invalid_manifest",
2071 format!(
2072 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2073 hello.protocol_ver, hello.manifest.protocol_ver
2074 ),
2075 )?]);
2076 }
2077
2078 if hello.manifest.module_id.trim().is_empty() {
2079 return Ok(vec![control_error_frame(
2080 &frame,
2081 "invalid_manifest",
2082 "manifest module_id must not be empty",
2083 )?]);
2084 }
2085
2086 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2087 Ok(negotiated_ver) => negotiated_ver,
2088 Err(message) => {
2089 return Ok(vec![control_error_frame(
2090 &frame,
2091 "version_unsupported",
2092 message,
2093 )?])
2094 }
2095 };
2096
2097 let swap_admission = self
2105 .supervisor
2106 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2107 if swap_admission == SwapHelloAdmission::Refused {
2108 warn!(
2109 module_id = %hello.manifest.module_id,
2110 connection_id = connection_id.get(),
2111 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2112 );
2113 return Ok(vec![control_error_frame(
2114 &frame,
2115 "swap_token_invalid",
2116 format!(
2117 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2118 hello.manifest.module_id
2119 ),
2120 )?]);
2121 }
2122 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2123
2124 if let Some(rejection) = (!swap_candidate)
2132 .then(|| {
2133 self.supervisor.reserved_hello_rejection(
2134 &hello.manifest.module_id,
2135 hello.launch_nonce.as_deref(),
2136 )
2137 })
2138 .flatten()
2139 {
2140 let message = match rejection {
2141 ReservedHelloRejection::Exact { module_id } => format!(
2142 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2143 ),
2144 ReservedHelloRejection::Prefix {
2145 prefix,
2146 owner_module_id,
2147 } => format!(
2148 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2149 hello.manifest.module_id
2150 ),
2151 };
2152 return Ok(vec![control_error_frame(
2153 &frame,
2154 "reserved_module",
2155 message,
2156 )?]);
2157 }
2158
2159 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2160 &hello.manifest.module_id,
2161 hello.manifest.capabilities.as_ref(),
2162 );
2163 if let Some(refusal) = reserved_capability_refusals.first() {
2164 let capability = refusal.capability.clone();
2165 let bound_module = refusal.claimants[0].clone();
2166 log_duplicate_claim_events(reserved_capability_refusals);
2167 return Ok(vec![control_error_frame(
2168 &frame,
2169 "reserved_capability",
2170 format!(
2171 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2172 capability, bound_module, hello.manifest.module_id
2173 ),
2174 )?]);
2175 }
2176
2177 if self
2180 .forwarding
2181 .connection_has_client_routes(connection_id)
2182 .map_err(RouterError::Forwarding)?
2183 {
2184 return Ok(vec![control_error_frame(
2185 &frame,
2186 "invalid_hello",
2187 "connection has open client routes and cannot also register as a module",
2188 )?]);
2189 }
2190
2191 self.hello_launch_nonces
2196 .lock()
2197 .unwrap_or_else(|poisoned| poisoned.into_inner())
2198 .record(connection_id, hello.launch_nonce.as_deref());
2199 let control_ops = effective_module_control_ops(hello.control_ops);
2200 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2203 if swap_candidate {
2204 return self.register_swap_candidate(
2205 connection_id,
2206 sink,
2207 &frame,
2208 hello.manifest,
2209 negotiated_ver,
2210 control_ops,
2211 hello_ack,
2212 );
2213 }
2214 let registration = match self.registry.register_with_control_ops(
2215 hello.manifest,
2216 negotiated_ver,
2217 connection_id,
2218 control_ops,
2219 ) {
2220 Ok(registration) => registration,
2221 Err(RegistryError::DuplicateModuleId { module_id }) => {
2222 return Ok(vec![control_error_frame(
2223 &frame,
2224 "duplicate_module_id",
2225 format!(
2226 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2227 ),
2228 )?])
2229 }
2230 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2231 return Ok(vec![control_error_frame(
2232 &frame,
2233 "invalid_module_id",
2234 err.to_string(),
2235 )?])
2236 }
2237 Err(err) => {
2238 return Ok(vec![control_error_frame(
2239 &frame,
2240 "registry_error",
2241 err.to_string(),
2242 )?])
2243 }
2244 };
2245
2246 let reply = if let Some(sink) = sink {
2247 let concurrency = manifest_concurrency(®istration.manifest);
2263 if let Err(err) = self.forwarding.register_module_connection_acked(
2264 connection_id,
2265 registration.manifest.module_id.clone(),
2266 negotiated_ver,
2267 concurrency,
2268 sink,
2269 hello_ack,
2270 ) {
2271 if matches!(
2275 self.deregister_connection(
2276 connection_id,
2277 RegistrationEndReason::RegistrationFailed,
2278 ),
2279 Ok(r) if !r.is_empty()
2280 ) {
2281 crate::supervise::notify_registration_release();
2282 }
2283 return Ok(vec![control_error_frame(
2284 &frame,
2285 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2286 "invalid_hello"
2287 } else {
2288 forwarding_error_code(&err)
2289 },
2290 err.to_string(),
2291 )?]);
2292 }
2293 Vec::new()
2294 } else {
2295 vec![hello_ack]
2298 };
2299
2300 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2309 info!(
2310 module_id = %registration.manifest.module_id,
2311 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2312 );
2313 }
2314
2315 self.apply_registration_capabilities(®istration);
2316
2317 info!(
2318 module_id = %registration.manifest.module_id,
2319 module_version = %registration.manifest.module_version,
2320 negotiated_ver,
2321 routable_provider = manifest_provides_routable_role(®istration.manifest),
2322 connection_id = connection_id.get(),
2323 "module registered"
2324 );
2325
2326 Ok(reply)
2327 }
2328
2329 #[allow(clippy::too_many_arguments)]
2339 fn register_swap_candidate(
2340 &self,
2341 connection_id: ConnectionId,
2342 sink: Option<crate::FrameSink>,
2343 frame: &Frame,
2344 manifest: ModuleManifest,
2345 negotiated_ver: u8,
2346 control_ops: Vec<String>,
2347 hello_ack: Frame,
2348 ) -> Result<Vec<Frame>, RouterError> {
2349 let module_id = manifest.module_id.clone();
2350 let registration = match self.registry.register_candidate_with_control_ops(
2351 manifest,
2352 negotiated_ver,
2353 connection_id,
2354 control_ops,
2355 ) {
2356 Ok(registration) => registration,
2357 Err(RegistryError::DuplicateModuleId { module_id }) => {
2358 return Ok(vec![control_error_frame(
2359 frame,
2360 "duplicate_module_id",
2361 format!(
2362 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2363 ),
2364 )?])
2365 }
2366 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2367 return Ok(vec![control_error_frame(
2368 frame,
2369 "invalid_module_id",
2370 err.to_string(),
2371 )?])
2372 }
2373 Err(err) => {
2374 return Ok(vec![control_error_frame(
2375 frame,
2376 "registry_error",
2377 err.to_string(),
2378 )?])
2379 }
2380 };
2381 let reply = if let Some(sink) = sink {
2382 let concurrency = manifest_concurrency(®istration.manifest);
2386 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2387 connection_id,
2388 module_id.clone(),
2389 negotiated_ver,
2390 concurrency,
2391 sink,
2392 hello_ack,
2393 ) {
2394 if matches!(
2395 self.deregister_connection(
2396 connection_id,
2397 RegistrationEndReason::RegistrationFailed,
2398 ),
2399 Ok(r) if !r.is_empty()
2400 ) {
2401 crate::supervise::notify_registration_release();
2402 }
2403 return Ok(vec![control_error_frame(
2404 frame,
2405 forwarding_error_code(&err),
2406 err.to_string(),
2407 )?]);
2408 }
2409 Vec::new()
2410 } else {
2411 vec![hello_ack]
2412 };
2413 self.supervisor.mark_swap_candidate_admitted(&module_id);
2414 info!(
2415 module_id = %module_id,
2416 module_version = %registration.manifest.module_version,
2417 negotiated_ver,
2418 ready = registration.ready,
2419 connection_id = connection_id.get(),
2420 "swap candidate registered; not routable until cutover"
2421 );
2422 Ok(reply)
2423 }
2424
2425 fn build_hello_ack(
2426 &self,
2427 frame: &Frame,
2428 negotiated_ver: u8,
2429 module_id: &str,
2430 ) -> Result<Frame, RouterError> {
2431 let ack = ModuleHelloAckBody {
2432 negotiated_ver,
2433 subc_ops: module_subc_ops(),
2434 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2435 storage: self
2436 .storage_config
2437 .as_ref()
2438 .map(|cfg| cfg.descriptor_for(module_id)),
2439 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2440 };
2441 let body = serde_json::to_vec(&ack).map_err(|err| {
2442 RouterError::backend(
2443 0,
2444 frame.header.corr,
2445 format!("failed to encode HELLO_ACK: {err}"),
2446 )
2447 })?;
2448
2449 Frame::build_with_version(
2450 negotiated_ver,
2451 FrameType::HelloAck,
2452 control_flags(),
2453 0,
2454 0,
2455 frame.header.corr,
2456 body,
2457 )
2458 .map_err(RouterError::FrameBuild)
2459 }
2460
2461 async fn handle_client_control_request(
2462 &self,
2463 ctx: &RouteCtx,
2464 frame: Frame,
2465 request: ClientControlRequest,
2466 ) -> Result<Vec<Frame>, RouterError> {
2467 match request {
2468 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2469 ClientControlRequest::CatalogList { module_id } => {
2470 self.handle_catalog_list(frame, module_id)
2471 }
2472 ClientControlRequest::RouteOpen {
2473 target,
2474 identity,
2475 consumer_identity,
2476 consumer_capabilities,
2477 role_versions,
2478 admission_facts,
2479 scope,
2480 } => {
2481 self.handle_route_open(
2482 ctx,
2483 frame,
2484 RouteOpenRequest {
2485 target,
2486 identity,
2487 consumer_identity,
2488 consumer_capabilities,
2489 role_versions,
2490 admission_facts,
2491 scope,
2492 },
2493 )
2494 .await
2495 }
2496 ClientControlRequest::RoutePoll {
2497 route_channel,
2498 route_epoch,
2499 kind,
2500 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2501 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2502 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2503 self.handle_supervisor_spawn_snapshot(frame)
2504 }
2505 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2506 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2507 }
2508 ClientControlRequest::SupervisorRestart {
2509 module_id,
2510 drain_timeout_ms,
2511 } => {
2512 self.log_supervisor_request_received(
2513 ctx,
2514 frame.header.corr,
2515 ops::SUPERVISOR_RESTART,
2516 Some(&module_id),
2517 None,
2518 )?;
2519 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms)
2520 .await
2521 }
2522 ClientControlRequest::SupervisorSwap {
2523 module_id,
2524 ready_timeout_ms,
2525 } => {
2526 self.log_supervisor_request_received(
2527 ctx,
2528 frame.header.corr,
2529 ops::SUPERVISOR_SWAP,
2530 Some(&module_id),
2531 None,
2532 )?;
2533 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms)
2534 .await
2535 }
2536 ClientControlRequest::SupervisorReload { module_id } => {
2537 self.log_supervisor_request_received(
2538 ctx,
2539 frame.header.corr,
2540 ops::SUPERVISOR_RELOAD,
2541 Some(&module_id),
2542 None,
2543 )?;
2544 self.handle_supervisor_reload(frame, module_id).await
2545 }
2546 ClientControlRequest::SupervisorRescan { preview } => {
2547 if !preview {
2548 self.log_supervisor_request_received(
2549 ctx,
2550 frame.header.corr,
2551 ops::SUPERVISOR_RESCAN,
2552 None,
2553 None,
2554 )?;
2555 }
2556 self.handle_supervisor_rescan(frame, preview).await
2557 }
2558 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2559 self.log_supervisor_request_received(
2560 ctx,
2561 frame.header.corr,
2562 ops::SUPERVISOR_RELEASE_RESERVED,
2563 Some(&module_id),
2564 None,
2565 )?;
2566 self.handle_supervisor_release_reserved(frame, module_id)
2567 .await
2568 }
2569 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2570 self.log_supervisor_request_received(
2571 ctx,
2572 frame.header.corr,
2573 ops::SUPERVISOR_SET_ENABLED,
2574 Some(&module_id),
2575 Some(enabled),
2576 )?;
2577 self.handle_supervisor_set_enabled(frame, module_id, enabled)
2578 .await
2579 }
2580 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2581 self.handle_supervisor_health_probe(frame, module_id).await
2582 }
2583 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2584 ClientControlRequest::SupervisorRoutes { module_id } => {
2585 self.handle_supervisor_routes(frame, module_id)
2586 }
2587 ClientControlRequest::SupervisorProvenance { module_id } => {
2588 self.handle_supervisor_provenance(frame, module_id).await
2589 }
2590 ClientControlRequest::SupervisorStderrTail {
2591 module_id,
2592 max_lines,
2593 max_bytes,
2594 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2595 ClientControlRequest::SupervisorTerminals { module_id } => {
2596 self.handle_supervisor_terminals(frame, module_id).await
2597 }
2598 }
2599 }
2600
2601 fn log_supervisor_request_received(
2602 &self,
2603 ctx: &RouteCtx,
2604 corr: u64,
2605 op: &'static str,
2606 module_id: Option<&str>,
2607 enabled: Option<bool>,
2608 ) -> Result<(), RouterError> {
2609 let caller = self
2610 .registry
2611 .get_module_by_connection(ctx.connection_id)
2612 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2613 .map(|registration| Principal::Reserved {
2614 module_id: registration.manifest.module_id,
2615 })
2616 .unwrap_or(Principal::Direct);
2617 let caller = principal_label(&caller);
2618 info!(
2621 target: "control",
2622 op,
2623 module_id,
2624 enabled,
2625 connection_id = ctx.connection_id.get(),
2626 caller = %caller,
2627 "supervisor request received"
2628 );
2629 Ok(())
2630 }
2631
2632 fn handle_module_control_request(
2633 &self,
2634 connection_id: ConnectionId,
2635 frame: Frame,
2636 request: ModuleControlRequestFromModule,
2637 ) -> Result<Vec<Frame>, RouterError> {
2638 match request {
2639 ModuleControlRequestFromModule::CatalogUpdate {
2640 provides,
2641 capabilities,
2642 ready,
2643 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2644 ModuleControlRequestFromModule::LiveRoots {} => {
2645 let registered = self
2646 .registry
2647 .get_module_by_connection(connection_id)
2648 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2649 let Some(registration) = registered else {
2650 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2651 };
2652 let response = self
2653 .forwarding
2654 .live_roots(®istration.manifest.module_id)
2655 .map_err(RouterError::Forwarding)?;
2656 Ok(vec![control_response_body_frame(
2657 &frame,
2658 &response,
2659 "ModuleControlResponseToModule::LiveRoots",
2660 )?])
2661 }
2662 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2663 self.handle_scope_sync(connection_id, frame, generation, scopes)
2664 }
2665 ModuleControlRequestFromModule::ScopeApply {
2666 generation,
2667 upsert,
2668 end,
2669 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2670 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2671 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2672 }
2673 }
2674 }
2675
2676 fn handle_operator_confirm(
2677 &self,
2678 ctx: &RouteCtx,
2679 frame: Frame,
2680 ) -> Result<Vec<Frame>, RouterError> {
2681 use crate::operator_confirm::{audit, Outcome};
2682 let registration = self
2683 .registry
2684 .get_module_by_connection(ctx.connection_id)
2685 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2686 let Some(registration) = registration else {
2687 let outcome = Outcome::refusal("not_registered");
2688 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2689 return Ok(vec![outcome.frame(&frame)]);
2690 };
2691 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2692 Ok(request) => request,
2693 Err(_) => {
2694 let outcome = Outcome::refusal("invalid_control_body");
2695 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2696 return Ok(vec![outcome.frame(&frame)]);
2697 }
2698 };
2699 let module_id = registration.manifest.module_id;
2700 let nonce = self
2704 .hello_launch_nonces
2705 .lock()
2706 .unwrap_or_else(|p| p.into_inner())
2707 .nonce(ctx.connection_id)
2708 .map(str::to_owned);
2709 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2710 self.supervisor
2711 .spawned_consumer_authorized(&module_id, nonce)
2712 });
2713 let confirms = self.forwarding.operator_confirms();
2714 self.forwarding
2715 .with_operator_route(
2716 ctx.connection_id,
2717 request.route_channel,
2718 request.route_epoch,
2719 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
2720 )
2721 .map_err(RouterError::Forwarding)
2722 }
2723
2724 fn handle_scope_sync(
2728 &self,
2729 connection_id: ConnectionId,
2730 frame: Frame,
2731 generation: u64,
2732 scopes: Vec<ScopeRecord>,
2733 ) -> Result<Vec<Frame>, RouterError> {
2734 self.handle_scope_change(connection_id, frame, generation, scopes, None)
2735 }
2736
2737 fn handle_scope_change(
2738 &self,
2739 connection_id: ConnectionId,
2740 frame: Frame,
2741 generation: u64,
2742 scopes: Vec<ScopeRecord>,
2743 end: Option<Vec<ScopeEnd>>,
2744 ) -> Result<Vec<Frame>, RouterError> {
2745 let is_apply = end.is_some();
2746 let Some(registration) = self
2747 .registry
2748 .get_module_by_connection(connection_id)
2749 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2750 else {
2751 return Ok(vec![control_error_frame(
2752 &frame,
2753 "not_registered",
2754 "scope.sync and scope.apply require an active module registration owned by this connection",
2755 )?]);
2756 };
2757 let owner = registration.manifest.module_id;
2758 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2759 let is_current_launch = |connection: ConnectionId| {
2760 self.hello_launch_nonces
2761 .lock()
2762 .unwrap_or_else(|poisoned| poisoned.into_inner())
2763 .presented(connection, current_nonce.as_deref())
2764 };
2765 let mut table = self
2770 .scopes
2771 .write()
2772 .unwrap_or_else(|poisoned| poisoned.into_inner());
2773 let outcome = match end {
2774 Some(end) => table.apply(
2775 &owner,
2776 connection_id,
2777 is_current_launch,
2778 generation,
2779 scopes,
2780 end,
2781 ),
2782 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
2783 };
2784 let drained = match &outcome {
2785 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
2786 Err(_) => Vec::new(),
2787 };
2788 let has_deadlines = outcome.is_ok() && table.has_deadlines();
2789 drop(table);
2790 if has_deadlines {
2791 self.scope_deadline_added.notify_one();
2792 }
2793 match outcome {
2794 Ok(applied) => {
2795 let counts = ScopeOutcomeCounts::of(&applied.results);
2796 info!(
2797 owner = %owner,
2798 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2799 generation,
2800 records = applied.results.len(),
2801 created = counts.created,
2802 replaced = counts.replaced,
2803 updated = counts.updated,
2804 unchanged = counts.unchanged,
2805 refused = counts.refused,
2806 ended = applied.ended.len(),
2807 tag_changes = applied.tag_changes.len(),
2808 routes_closed = drained.len(),
2809 "scope change accepted"
2810 );
2811 for refused in applied
2817 .results
2818 .iter()
2819 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2820 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2821 {
2822 warn!(
2823 owner = %owner,
2824 generation,
2825 scope_ref = %refused.scope_ref,
2826 scope_epoch = refused.scope_epoch,
2827 code = refused.code.as_deref().unwrap_or(""),
2828 "scope record refused"
2829 );
2830 }
2831 self.close_scope_drained_routes(drained);
2832 let response = if is_apply {
2833 ModuleControlResponseToModule::ScopeApply {
2834 generation,
2835 results: applied.results,
2836 end_results: applied.end_results,
2837 ended: applied.ended,
2838 }
2839 } else {
2840 ModuleControlResponseToModule::ScopeSync {
2841 generation,
2842 results: applied.results,
2843 ended: applied.ended,
2844 }
2845 };
2846 Ok(vec![control_response_body_frame(
2847 &frame,
2848 &response,
2849 "scope change response",
2850 )?])
2851 }
2852 Err(refusal) => {
2853 info!(
2854 owner = %owner,
2855 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2856 generation,
2857 code = refusal.code,
2858 "scope change refused"
2859 );
2860 Ok(vec![control_error_frame(
2861 &frame,
2862 refusal.code,
2863 refusal.message,
2864 )?])
2865 }
2866 }
2867 }
2868
2869 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2875 if drained.is_empty() {
2876 return;
2877 }
2878 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2879 BTreeMap::new();
2880 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2881 for route in drained {
2882 warn!(
2883 module_id = %route.module_id,
2884 reason = ?route.reason,
2885 client_connection_id = route.client.connection_id.get(),
2886 route_channel = route.client.channel,
2887 "closing route because its scope changed"
2888 );
2889 pushes
2890 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2891 .or_insert_with(|| (route.reason, Vec::new()))
2892 .1
2893 .push(EndpointRoute {
2894 goodbye_target: route.client.clone(),
2895 principal: Principal::Unverified,
2896 bound_at: Instant::now(),
2897 draining: false,
2898 drain_reason: None,
2899 });
2900 goodbyes.push(route.module);
2901 goodbyes.push(route.client);
2902 }
2903 for ((module_id, _), (reason, routes)) in pushes {
2904 send_route_control_pushes(
2905 &self.forwarding,
2906 routes,
2907 ClientControlPush::RouteClosed {
2908 module_id,
2909 channels: Vec::new(),
2910 reason,
2911 drained: false,
2912 abandoned: 0,
2913 excluded_subscriptions: 0,
2914 terminal: Some(false),
2915 },
2916 );
2917 }
2918 self.emit_route_goodbyes(goodbyes);
2919 }
2920
2921 fn handle_scope_describe(
2924 &self,
2925 connection_id: ConnectionId,
2926 frame: Frame,
2927 owner: Principal,
2928 scope_ref: String,
2929 ) -> Result<Vec<Frame>, RouterError> {
2930 let registered = self
2931 .registry
2932 .get_module_by_connection(connection_id)
2933 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2934 if registered.is_none() {
2935 return Ok(vec![control_error_frame(
2936 &frame,
2937 "not_registered",
2938 "scope.describe requires an active module registration owned by this connection",
2939 )?]);
2940 }
2941 let description = self
2942 .scopes
2943 .read()
2944 .unwrap_or_else(|poisoned| poisoned.into_inner())
2945 .describe(&owner, &scope_ref);
2946 let owner_configured = match &owner {
2947 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
2952 _ => false,
2953 };
2954 let response = ModuleControlResponseToModule::ScopeDescribe {
2955 status: description.status,
2956 scope_epoch: description.scope_epoch,
2957 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
2958 owner_synced: description.owner_synced,
2959 owner_configured,
2960 scope: description.stamp,
2961 };
2962 Ok(vec![control_response_body_frame(
2963 &frame,
2964 &response,
2965 "ModuleControlResponseToModule::ScopeDescribe",
2966 )?])
2967 }
2968
2969 fn handle_catalog_update(
2970 &self,
2971 connection_id: ConnectionId,
2972 frame: Frame,
2973 provides: Vec<ProviderRole>,
2974 capabilities: Option<CapabilityDeclarations>,
2975 ready: Option<bool>,
2976 ) -> Result<Vec<Frame>, RouterError> {
2977 self.refresh_capability_requirements();
2978 let Some(registration) = self
2979 .registry
2980 .get_module_by_connection(connection_id)
2981 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2982 else {
2983 return Ok(vec![control_error_frame(
2984 &frame,
2985 "not_registered",
2986 "catalog.update requires an active module registration owned by this connection",
2987 )?]);
2988 };
2989
2990 if let Some(message) =
2991 catalog_update_frozen_field_message(®istration.manifest, &provides)
2992 {
2993 return Ok(vec![control_error_frame(
2994 &frame,
2995 "catalog_update_frozen_field",
2996 message,
2997 )?]);
2998 }
2999
3000 let mut candidate = registration.manifest.clone();
3001 candidate.provides = provides.clone();
3002 candidate.capabilities = capabilities
3003 .clone()
3004 .or_else(|| registration.manifest.capabilities.clone());
3005 if let Err(err) = candidate.validate_capability_grammar() {
3006 return Ok(vec![control_error_frame(
3007 &frame,
3008 "invalid_capability_grammar",
3009 err.to_string(),
3010 )?]);
3011 }
3012
3013 let mut conflicts = self
3016 .capability_evaluator
3017 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3018 if let Some(conflict) = conflicts.first() {
3019 let message = format!(
3020 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3021 conflict.capability, conflict.claimants[0], candidate.module_id
3022 );
3023 for conflict in &mut conflicts {
3024 conflict.source = DuplicateClaimSource::CatalogUpdate;
3025 }
3026 log_duplicate_claim_events(conflicts);
3027 return Ok(vec![control_error_frame(
3028 &frame,
3029 "reserved_capability",
3030 message,
3031 )?]);
3032 }
3033
3034 let updated = self
3035 .registry
3036 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3037 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3038 if updated.is_none() {
3039 return Ok(vec![control_error_frame(
3040 &frame,
3041 "not_registered",
3042 "catalog.update requires an active module registration owned by this connection",
3043 )?]);
3044 }
3045 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3046 log_duplicate_claim_events(
3047 self.capability_evaluator
3048 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3049 );
3050 }
3051 if capability_census_trigger(
3052 registration.manifest.capabilities.as_ref(),
3053 updated
3054 .as_ref()
3055 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3056 ) {
3057 self.enforce_capability_denies();
3058 }
3059 self.refresh_capability_requirements();
3060
3061 let response = ModuleControlResponseToModule::CatalogUpdate {};
3062 control_response_body_frame(
3063 &frame,
3064 &response,
3065 "ModuleControlResponseToModule::CatalogUpdate",
3066 )
3067 .map(|frame| vec![frame])
3068 }
3069
3070 pub(crate) fn with_activity_ledger_config(
3071 mut self,
3072 run_dir: &Path,
3073 incarnation: String,
3074 config: crate::activity::ActivityConfig,
3075 ) -> Self {
3076 self.activity = Some(crate::activity::ActivityLedger::open_config(
3077 run_dir.join("activity.jsonl"),
3078 incarnation,
3079 config,
3080 ));
3081 self
3082 }
3083
3084 pub(crate) async fn configure_activity(
3085 &self,
3086 config: crate::activity::ActivityConfig,
3087 ) -> Result<(), &'static str> {
3088 match &self.activity {
3089 Some(ledger) => ledger.configure(config).await,
3090 None => Ok(()),
3091 }
3092 }
3093
3094 fn cancel_activity_subscription(
3095 &self,
3096 ctx: &RouteCtx,
3097 frame: &Frame,
3098 ) -> Result<bool, RouterError> {
3099 let task = self
3100 .activity_streams
3101 .lock()
3102 .unwrap_or_else(|p| p.into_inner())
3103 .remove(&(ctx.connection_id, frame.header.corr));
3104 let Some(task) = task else {
3105 return Ok(false);
3106 };
3107 task.abort();
3108 let end = Frame::build_with_version(
3109 response_version(frame),
3110 FrameType::StreamEnd,
3111 control_flags(),
3112 0,
3113 0,
3114 frame.header.corr,
3115 Vec::new(),
3116 )
3117 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3118 let sink = ctx.egress.clone();
3119 tokio::spawn(async move {
3120 let _ = sink.send(end).await;
3121 });
3122 Ok(true)
3123 }
3124
3125 async fn handle_activity_read(
3126 &self,
3127 ctx: &RouteCtx,
3128 frame: Frame,
3129 ) -> Result<Vec<Frame>, RouterError> {
3130 use subc_control::{
3131 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3132 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3133 };
3134 #[derive(Deserialize)]
3135 struct Identity {
3136 #[serde(default)]
3137 consumer_identity: Option<ConsumerIdentity>,
3138 }
3139 let principal = match self
3143 .forwarding
3144 .module_id_for_connection(ctx.connection_id)
3145 .map_err(RouterError::Forwarding)?
3146 {
3147 Some(module_id) => Ok(Principal::Reserved { module_id }),
3148 None => {
3149 let identity: Identity = match serde_json::from_slice(&frame.body) {
3150 Ok(identity) => identity,
3151 Err(_) => {
3152 return Ok(vec![control_error_frame(
3153 &frame,
3154 ACTIVITY_MALFORMED,
3155 "malformed activity read",
3156 )?])
3157 }
3158 };
3159 self.route_open_principal(&frame, identity.consumer_identity)?
3160 }
3161 };
3162 let admitted = matches!(principal, Ok(Principal::Direct))
3163 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3164 if !admitted {
3165 return Ok(vec![control_error_frame(
3166 &frame,
3167 ACTIVITY_READ_FORBIDDEN,
3168 "activity reads require an operator or trusted mirror principal",
3169 )?]);
3170 }
3171 let Some(ledger) = &self.activity else {
3172 return Ok(vec![control_error_frame(
3173 &frame,
3174 ACTIVITY_UNAVAILABLE,
3175 "activity ledger unavailable",
3176 )?]);
3177 };
3178 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3179 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3180 {
3181 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3182 Ok(request) => ledger.query(request).await,
3183 Err(_) => Err(ACTIVITY_MALFORMED),
3184 };
3185 return match result {
3186 Ok(reply) => Ok(vec![control_response_body_frame(
3187 &frame,
3188 &reply,
3189 ops::ACTIVITY_QUERY,
3190 )?]),
3191 Err(code) => Ok(vec![control_error_frame(
3192 &frame,
3193 code,
3194 "activity query refused",
3195 )?]),
3196 };
3197 }
3198 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3199 Ok(request) => request,
3200 Err(_) => {
3201 return Ok(vec![control_error_frame(
3202 &frame,
3203 ACTIVITY_MALFORMED,
3204 "malformed activity subscription",
3205 )?])
3206 }
3207 };
3208 let mut subscription = match ledger.subscribe(request.since).await {
3209 Ok(subscription) => subscription,
3210 Err(code) => {
3211 return Ok(vec![control_error_frame(
3212 &frame,
3213 code,
3214 "activity subscription refused",
3215 )?])
3216 }
3217 };
3218 let key = (ctx.connection_id, frame.header.corr);
3219 let sink = ctx.egress.clone();
3220 let mut streams = self
3221 .activity_streams
3222 .lock()
3223 .unwrap_or_else(|p| p.into_inner());
3224 streams.retain(|_, task| !task.is_finished());
3225 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3226 return Ok(vec![control_error_frame(
3227 &frame,
3228 "duplicate_subscription",
3229 "activity subscription correlation id already in use",
3230 )?]);
3231 }
3232 let task = tokio::spawn(async move {
3233 loop {
3234 let entry = if let Some(entry) = subscription.replay.pop_front() {
3235 Some(entry)
3236 } else {
3237 subscription.receiver.recv().await
3238 };
3239 let Some(entry) = entry else {
3240 break;
3241 };
3242 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3243 return;
3244 };
3245 let Ok(next) = Frame::build_with_version(
3246 response_version(&frame),
3247 FrameType::StreamData,
3248 control_flags(),
3249 0,
3250 0,
3251 frame.header.corr,
3252 body,
3253 ) else {
3254 return;
3255 };
3256 if sink.send(next).await.is_err() {
3257 return;
3258 }
3259 }
3260 if let Some(code) = subscription.terminal_code() {
3263 if let Ok(end) =
3264 control_error_frame(&frame, code, "activity subscriber fell behind")
3265 {
3266 let _ = sink.send(end).await;
3267 }
3268 }
3269 });
3270 streams.insert(key, task.abort_handle());
3271 Ok(Vec::new())
3272 }
3273
3274 async fn handle_activity_record(
3275 &self,
3276 ctx: &RouteCtx,
3277 frame: Frame,
3278 ) -> Result<Vec<Frame>, RouterError> {
3279 use subc_control::{
3280 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3281 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3282 };
3283 let result = async {
3284 let module = self
3285 .forwarding
3286 .module_id_for_connection(ctx.connection_id)
3287 .ok()
3288 .flatten()
3289 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3290 let request: ActivityRecordRequest =
3291 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3292 crate::activity::validate(&module, &request)?;
3293 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3294 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3295 let at_ms = std::time::SystemTime::now()
3296 .duration_since(std::time::UNIX_EPOCH)
3297 .unwrap_or_default()
3298 .as_millis() as u64;
3299 ledger.record(module, request, actor, at_ms).await
3300 }
3301 .await;
3302 match result {
3303 Ok(entry) => Ok(vec![control_response_body_frame(
3304 &frame,
3305 &ActivityRecordResponse { seq: entry.seq },
3306 "activity.record",
3307 )?]),
3308 Err(code) => Ok(vec![control_error_frame(
3309 &frame,
3310 code,
3311 "activity recording refused",
3312 )?]),
3313 }
3314 }
3315
3316 fn activity_actor(
3317 &self,
3318 connection: ConnectionId,
3319 module: &str,
3320 on: Option<&subc_control::ActivityOn>,
3321 ) -> Result<subc_control::ActivityActor, &'static str> {
3322 use crate::forwarding::{DataRoute, DataRouteState};
3323 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3324 let principal = |id: &str| Principal::Reserved {
3325 module_id: id.to_owned(),
3326 };
3327 let mut actor = ActivityActor {
3328 principal: principal(module),
3329 scope_owner: None,
3330 agent_id: None,
3331 run_id: None,
3332 flow_id: None,
3333 project_root: None,
3334 };
3335 let scope = match on {
3336 None => return Ok(actor),
3337 Some(ActivityOn::Scope(scope)) => {
3338 let selector = subc_protocol::scope::ScopeSelector {
3339 owner: principal(module),
3340 scope_ref: scope.scope_ref.clone(),
3341 scope_epoch: Some(scope.epoch),
3342 };
3343 self.scopes
3344 .read()
3345 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3346 .admit(&principal(module), module, &selector, true)
3347 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3348 }
3349 Some(ActivityOn::Route(route)) => {
3350 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3351 .forwarding
3352 .lookup_data_route(connection, route.channel, route.epoch)
3353 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3354 else {
3355 return Err(ACTIVITY_ACTOR_UNKNOWN);
3356 };
3357 if binding.module_id != module {
3358 return Err(ACTIVITY_ACTOR_UNKNOWN);
3359 }
3360 actor.principal = binding.principal.clone();
3361 actor.project_root = binding
3362 .project_root
3363 .as_ref()
3364 .map(|root| root.as_path().to_path_buf());
3365 let Some(bound) = &binding.scope else {
3366 return Ok(actor);
3367 };
3368 let owner = principal(&bound.owner);
3369 let selector = subc_protocol::scope::ScopeSelector {
3370 owner: owner.clone(),
3371 scope_ref: bound.scope_ref.clone(),
3372 scope_epoch: Some(bound.tag.scope_epoch),
3373 };
3374 let admitted = self
3375 .scopes
3376 .read()
3377 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3378 .admit(&owner, module, &selector, true)
3379 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3380 if admitted.tag != bound.tag {
3381 return Err(ACTIVITY_ACTOR_UNKNOWN);
3382 }
3383 admitted
3384 }
3385 };
3386 actor.scope_owner = Some(scope.stamp.owner);
3387 actor.agent_id = scope.stamp.attributes.agent_id;
3388 actor.run_id = scope.stamp.attributes.run_id;
3389 actor.flow_id = scope.stamp.attributes.flow_id;
3390 Ok(actor)
3391 }
3392
3393 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3394 self.refresh_capability_requirements();
3395 let mut counters = self.counters.snapshot();
3401 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3402 obj.insert(
3403 "activity_unparseable_lines".into(),
3404 ledger.read_errors().into(),
3405 );
3406 }
3407 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3408 self.forwarding.client_route_concentration(),
3409 counters.as_object_mut(),
3410 ) {
3411 obj.insert(
3412 "client_connections_with_routes".into(),
3413 connections_with_routes.into(),
3414 );
3415 obj.insert("max_routes_on_one_connection".into(), max.into());
3416 }
3417 if let (Some(open_breakers), Some(obj)) = (
3423 self.route_bind_breakers.open_snapshot(),
3424 counters.as_object_mut(),
3425 ) {
3426 obj.insert("route_bind_breakers_open".into(), open_breakers);
3427 }
3428 let response = ClientControlResponse::ServerDescribe {
3429 protocol_ver: PROTOCOL_VERSION,
3430 subc_ops: subc_ops(),
3431 capabilities: self.subc_capabilities.as_ref().to_vec(),
3432 connected_clients: self.connected_clients.count(),
3433 counters: Some(counters),
3434 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3435 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3436 capability_requirements: self.capability_requirement_statuses(),
3437 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3438 };
3439 Ok(vec![control_response_body_frame(
3440 &frame,
3441 &response,
3442 "ClientControlResponse::ServerDescribe",
3443 )?])
3444 }
3445
3446 fn handle_catalog_list(
3447 &self,
3448 frame: Frame,
3449 module_id: Option<String>,
3450 ) -> Result<Vec<Frame>, RouterError> {
3451 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3452 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3453 })?;
3454 let entries = modules
3455 .into_iter()
3456 .filter(|registration| {
3457 module_id
3458 .as_deref()
3459 .map(|wanted| registration.manifest.module_id == wanted)
3460 .unwrap_or(true)
3461 })
3462 .map(|registration| {
3463 let not_ready = self.not_ready_reason(®istration);
3464 let roles = registration.manifest.provides;
3465 CatalogEntry::new(
3466 registration.manifest.module_id,
3467 roles,
3468 registration.control_ops,
3469 )
3470 .with_ready(not_ready.is_none())
3471 .with_not_ready(not_ready)
3472 .with_module_version(Some(registration.manifest.module_version))
3473 .with_capabilities(registration.manifest.capabilities)
3474 .with_self_signals(registration.manifest.self_signals)
3475 .with_events(registration.manifest.events)
3476 })
3477 .collect();
3478 let response = ClientControlResponse::CatalogList {
3479 generation,
3480 modules: entries,
3481 subc_ops: subc_ops(),
3482 };
3483 Ok(vec![control_response_body_frame(
3484 &frame,
3485 &response,
3486 "ClientControlResponse::CatalogList",
3487 )?])
3488 }
3489
3490 fn route_open_principal(
3491 &self,
3492 frame: &Frame,
3493 consumer_identity: Option<ConsumerIdentity>,
3494 ) -> Result<Result<Principal, Frame>, RouterError> {
3495 let Some(consumer_identity) = consumer_identity else {
3496 return Ok(Ok(Principal::Direct));
3497 };
3498
3499 if self.supervisor.spawned_consumer_authorized(
3500 &consumer_identity.module_id,
3501 &consumer_identity.launch_nonce,
3502 ) {
3503 return Ok(Ok(Principal::Reserved {
3504 module_id: consumer_identity.module_id,
3505 }));
3506 }
3507
3508 Ok(Err(control_error_frame(
3509 frame,
3510 "bad_consumer_identity",
3511 format!(
3512 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3513 consumer_identity.module_id
3514 ),
3515 )?))
3516 }
3517
3518 fn route_open_refusal_frame(
3525 &self,
3526 ctx: &RouteCtx,
3527 frame: &Frame,
3528 module_id: &str,
3529 reason: &'static str,
3530 code: &'static str,
3531 message: impl Into<String>,
3532 ) -> Result<Frame, RouterError> {
3533 self.observe_route_open_refusal(ctx, module_id, reason, code);
3534 control_error_frame(frame, code, message.into())
3535 }
3536
3537 fn route_open_breaker_refusal_frame(
3555 &self,
3556 ctx: &RouteCtx,
3557 frame: &Frame,
3558 module_id: &str,
3559 consecutive_timeouts: u32,
3560 retry_in: Duration,
3561 probe_in_flight: bool,
3562 ) -> Result<Frame, RouterError> {
3563 self.counters
3564 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3565 debug!(
3566 target: "control",
3567 code = "module_timeout",
3568 module_id = ?module_id,
3569 connection_id = ctx.connection_id.get(),
3570 consecutive_timeouts,
3571 retry_in_ms = retry_in.as_millis() as u64,
3572 probe_in_flight,
3573 "route.open refused by open bind-relay breaker"
3574 );
3575 let detail = if probe_in_flight {
3582 "one test route is already being tried; retry once it settles".to_string()
3583 } else {
3584 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3585 };
3586 control_error_frame(
3587 frame,
3588 "module_timeout",
3589 format!(
3590 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3591 timed out in a row); {detail}; its established routes are unaffected"
3592 ),
3593 )
3594 }
3595
3596 fn observe_route_open_refusal(
3609 &self,
3610 ctx: &RouteCtx,
3611 module_id: &str,
3612 reason: &'static str,
3613 code: &'static str,
3614 ) {
3615 self.counters.increment_route_open_refused(code);
3616 info!(
3617 target: "control",
3618 code,
3619 reason,
3620 module_id = ?module_id,
3621 connection_id = ctx.connection_id.get(),
3622 "route.open refused"
3623 );
3624 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3625 self.route_outages.record_not_serving(module_id, reason);
3626 }
3627 }
3628
3629 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3674 self.route_outages.record_accepted(module_id);
3675 self.counters.increment_route_open_accepted(principal);
3676 info!(
3677 target: "control",
3678 principal,
3679 module_id,
3680 connection_id = ctx.connection_id.get(),
3681 "route.open accepted"
3682 );
3683 }
3684
3685 fn supervised_absent_route_open_refusal_frame(
3686 &self,
3687 ctx: &RouteCtx,
3688 frame: &Frame,
3689 module_id: &str,
3690 code: &'static str,
3691 status: &crate::supervise::ModuleStatus,
3692 ) -> Result<Frame, RouterError> {
3693 self.counters.increment_route_open_refused(code);
3694 info!(
3695 target: "control",
3696 code,
3697 reason = "supervised_not_registered",
3698 module_id = ?module_id,
3699 connection_id = ctx.connection_id.get(),
3700 state = %status.state,
3701 enabled = status.enabled,
3702 live = status.live,
3703 "route.open refused"
3704 );
3705 self.route_outages
3709 .record_not_serving(module_id, "supervised_not_registered");
3710 let message = if let Some(pid) = status.waiting_for_previous_pid {
3711 format!("waiting: previous instance (pid {pid}) is still running but hidden; it cannot be identified")
3712 } else {
3713 format!(
3714 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3715 status.state, status.enabled, status.live
3716 )
3717 };
3718 control_error_frame(frame, code, message)
3719 }
3720
3721 async fn handle_route_open(
3722 &self,
3723 ctx: &RouteCtx,
3724 frame: Frame,
3725 request: RouteOpenRequest,
3726 ) -> Result<Vec<Frame>, RouterError> {
3727 let RouteOpenRequest {
3728 target,
3729 mut identity,
3730 consumer_identity,
3731 consumer_capabilities,
3732 role_versions,
3733 admission_facts,
3734 scope,
3735 } = request;
3736 let target_module_id = target_module_id(&target).to_string();
3737 if self
3738 .registry
3739 .get_module_by_connection(ctx.connection_id)
3740 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3741 .is_some()
3742 {
3743 return Ok(vec![control_error_frame(
3744 &frame,
3745 "invalid_request",
3746 "module connections cannot open client routes",
3747 )?]);
3748 }
3749 debug!(
3750 connection_id = ctx.connection_id.get(),
3751 corr = frame.header.corr,
3752 module_id = %target_module_id,
3753 "handling route.open"
3754 );
3755
3756 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3762 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3763 self.observe_route_open_refusal(
3764 ctx,
3765 &target_module_id,
3766 "invalid_role_versions",
3767 error_codes::INVALID_REQUEST,
3768 );
3769 return Ok(vec![control_error_body_frame(
3770 &frame,
3771 ErrorBody {
3772 code: error_codes::INVALID_REQUEST.to_string(),
3773 message: error.to_string(),
3774 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3775 },
3776 )?]);
3777 }
3778
3779 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
3780 if status.waiting_for_previous_pid.is_some() {
3781 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3782 ctx,
3783 &frame,
3784 &target_module_id,
3785 "module_warming",
3786 &status,
3787 )?]);
3788 }
3789 }
3790
3791 let Some(registration) = self
3810 .registry
3811 .get_module(&target_module_id)
3812 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3813 else {
3814 if let Some((status, warming)) =
3815 self.supervisor_status(&target_module_id, frame.header.corr)?
3816 {
3817 if status.protocol == ModuleProtocol::None {
3824 return Ok(vec![self.route_open_refusal_frame(
3825 ctx,
3826 &frame,
3827 &target_module_id,
3828 "protocol_none",
3829 error_codes::MODULE_NO_PROTOCOL,
3830 format!(
3831 "module_id '{target_module_id}' is declared protocol: none; \
3832 it speaks no subc wire and serves no routes"
3833 ),
3834 )?]);
3835 }
3836 let code = if warming {
3837 "module_warming"
3838 } else {
3839 "target_unavailable"
3840 };
3841 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3842 ctx,
3843 &frame,
3844 &target_module_id,
3845 code,
3846 &status,
3847 )?]);
3848 }
3849 if let Some(removed_ago_ms) =
3850 self.supervisor.removal_tombstone_age_ms(&target_module_id)
3851 {
3852 return Ok(vec![self.route_open_refusal_frame(
3853 ctx,
3854 &frame,
3855 &target_module_id,
3856 "removed",
3857 error_codes::MODULE_REMOVED,
3858 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3859 )?]);
3860 }
3861 return Ok(vec![self.route_open_refusal_frame(
3862 ctx,
3863 &frame,
3864 &target_module_id,
3865 "not_registered",
3866 error_codes::UNKNOWN_MODULE,
3867 format!("module_id '{target_module_id}' is not registered"),
3868 )?]);
3869 };
3870
3871 if !registration.ready {
3875 self.counters
3876 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3877 info!(
3878 target: "control",
3879 code = error_codes::MODULE_WARMING,
3880 module_id = ?target_module_id,
3881 connection_id = ctx.connection_id.get(),
3882 reason = "declared_not_ready",
3883 "route.open refused"
3884 );
3885 self.route_outages
3888 .record_not_serving(&target_module_id, "declared_not_ready");
3889 return Ok(vec![control_error_body_frame(
3890 &frame,
3891 ErrorBody {
3892 code: error_codes::MODULE_WARMING.to_string(),
3893 message: format!(
3894 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
3895 ),
3896 detail: Some(serde_json::json!({
3897 "reason": "declared_not_ready"
3898 })),
3899 },
3900 )?]);
3901 }
3902
3903 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
3922 self.counters
3923 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
3924 info!(
3925 target: "control",
3926 code = error_codes::MODULE_WARMING,
3927 module_id = ?target_module_id,
3928 connection_id = ctx.connection_id.get(),
3929 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3930 capability = %capability,
3931 "route.open refused"
3932 );
3933 return Ok(vec![control_error_body_frame(
3934 &frame,
3935 ErrorBody {
3936 code: error_codes::MODULE_WARMING.to_string(),
3937 message: format!(
3938 "module_id '{target_module_id}' requires capability '{capability}', \
3939 which no registered module provides; retry"
3940 ),
3941 detail: Some(serde_json::json!({
3942 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
3943 "capability": capability,
3944 })),
3945 },
3946 )?]);
3947 }
3948
3949 if !target_has_required_role(&target, ®istration.manifest.provides) {
3950 return Ok(vec![self.route_open_refusal_frame(
3951 ctx,
3952 &frame,
3953 &target_module_id,
3954 "role_not_provided",
3955 "target_unavailable",
3956 format!("module_id '{target_module_id}' does not provide the requested target"),
3957 )?]);
3958 }
3959
3960 if registration.state != ChannelState::Active {
3961 return Ok(vec![self.route_open_refusal_frame(
3962 ctx,
3963 &frame,
3964 &target_module_id,
3965 "registration_not_active",
3966 "target_unavailable",
3967 format!("module_id '{target_module_id}' is not active"),
3968 )?]);
3969 }
3970
3971 if self
3972 .forwarding
3973 .module_is_draining(&target_module_id)
3974 .map_err(RouterError::Forwarding)?
3975 {
3976 return Ok(vec![self.route_open_refusal_frame(
3977 ctx,
3978 &frame,
3979 &target_module_id,
3980 "reloading",
3981 "module_reloading",
3982 format!("module_id '{target_module_id}' is reloading"),
3983 )?]);
3984 }
3985
3986 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
3987 process_liveness.process_live(&target_module_id) == Some(false)
3988 }) {
3989 if process_liveness.process_replacing(&target_module_id) {
3996 return Ok(vec![self.route_open_refusal_frame(
3997 ctx,
3998 &frame,
3999 &target_module_id,
4000 "reloading",
4001 "module_reloading",
4002 format!("module_id '{target_module_id}' is reloading"),
4003 )?]);
4004 }
4005 return Ok(vec![self.route_open_refusal_frame(
4006 ctx,
4007 &frame,
4008 &target_module_id,
4009 "supervisor_not_live",
4010 "target_unavailable",
4011 format!("module_id '{target_module_id}' is not live"),
4012 )?]);
4013 }
4014
4015 if !self
4016 .forwarding
4017 .has_live_module_connection(&target_module_id)
4018 .map_err(RouterError::Forwarding)?
4019 {
4020 return Ok(vec![self.route_open_refusal_frame(
4021 ctx,
4022 &frame,
4023 &target_module_id,
4024 "no_forwarding_connection",
4025 "target_unavailable",
4026 format!("module_id '{target_module_id}' has no live forwarding connection"),
4027 )?]);
4028 }
4029
4030 if let Some(error) =
4031 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4032 {
4033 self.observe_route_open_refusal(
4034 ctx,
4035 &target_module_id,
4036 "op_not_allowed",
4037 "op_not_allowed",
4038 );
4039 return Ok(vec![error]);
4040 }
4041
4042 let principal = match self.route_open_principal(&frame, consumer_identity)? {
4043 Ok(principal) => principal,
4044 Err(error) => {
4045 self.observe_route_open_refusal(
4046 ctx,
4047 &target_module_id,
4048 "bad_consumer_identity",
4049 "bad_consumer_identity",
4050 );
4051 return Ok(vec![error]);
4052 }
4053 };
4054
4055 if let Principal::Reserved {
4059 module_id: opening_module_id,
4060 } = &principal
4061 {
4062 if let Some(opening_registration) = self
4063 .registry
4064 .get_module(opening_module_id)
4065 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4066 {
4067 if let Some(capability) =
4068 denied_capability(&opening_registration.manifest, ®istration.manifest)
4069 {
4070 warn!(
4071 opening_module_id,
4072 target_module_id,
4073 capability,
4074 "refusing route.open because an attested capability deny edge matches"
4075 );
4076 return Ok(vec![self.route_open_refusal_frame(
4077 ctx,
4078 &frame,
4079 &target_module_id,
4080 "capability_deny_edge",
4081 "capability_forbidden",
4082 format!(
4083 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4084 ),
4085 )?]);
4086 }
4087 }
4088 }
4089
4090 if admission_facts.is_some() {
4091 let carrier_matches = matches!(
4092 &principal,
4093 Principal::Reserved { module_id }
4094 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4095 );
4096 if !carrier_matches {
4097 return Ok(vec![self.route_open_refusal_frame(
4098 ctx,
4099 &frame,
4100 &target_module_id,
4101 "admission_facts_carrier_not_permitted",
4102 "admission_facts_not_permitted",
4103 "admission facts may only be carried by the configured reserved module",
4104 )?]);
4105 }
4106
4107 let target_allowed = self
4108 .admission_facts_targets
4109 .as_ref()
4110 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4111 if !target_allowed {
4112 return Ok(vec![self.route_open_refusal_frame(
4113 ctx,
4114 &frame,
4115 &target_module_id,
4116 "admission_facts_target_not_listed",
4117 "admission_facts_target_not_allowed",
4118 format!(
4119 "admission facts are not permitted for target module_id '{target_module_id}'"
4120 ),
4121 )?]);
4122 }
4123
4124 }
4128
4129 let (bound_scope, mut scope_stamp) = match scope {
4135 None => (None, None),
4136 Some(selector) => {
4137 let owner_configured = match &selector.owner {
4138 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4144 _ => false,
4145 };
4146 let admitted = self
4147 .scopes
4148 .read()
4149 .unwrap_or_else(|poisoned| poisoned.into_inner())
4150 .admit(&principal, &target_module_id, &selector, owner_configured)
4151 .and_then(|admission| {
4152 crate::scopes::check_target_flow_support(
4153 &admission.stamp,
4154 &target_module_id,
4155 registration.manifest.capabilities.as_ref(),
4156 )?;
4157 crate::scopes::check_target_agent_run_support(
4158 &admission.stamp,
4159 &target_module_id,
4160 registration.manifest.capabilities.as_ref(),
4161 )?;
4162 Ok(admission)
4163 });
4164 match admitted {
4165 Ok(admission) => (
4166 Some(BoundScope {
4167 owner: admission.owner,
4168 scope_ref: admission.stamp.scope_ref.clone(),
4169 tag: admission.tag,
4170 }),
4171 Some(admission.stamp),
4172 ),
4173 Err(refusal) => {
4174 return Ok(vec![self.route_open_refusal_frame(
4175 ctx,
4176 &frame,
4177 &target_module_id,
4178 refusal.code,
4179 refusal.code,
4180 refusal.message,
4181 )?]);
4182 }
4183 }
4184 }
4185 };
4186
4187 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4209 Ok(project_root) => project_root,
4210 Err(err) => {
4211 return Ok(vec![control_error_frame(
4212 &frame,
4213 "invalid_project_root",
4214 err.to_string(),
4215 )?])
4216 }
4217 };
4218 identity.project_root = project_root.as_path().to_path_buf();
4219
4220 let _concurrency_guard = match self
4231 .route_bind_concurrency
4232 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4233 {
4234 Ok(guard) => guard,
4235 Err(in_flight) => {
4236 return Ok(vec![self.route_open_target_capacity_refusal(
4237 ctx,
4238 &frame,
4239 &target_module_id,
4240 in_flight,
4241 )?]);
4242 }
4243 };
4244
4245 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4248 RouteBindAdmission::Admitted { guard, probe } => {
4249 if probe {
4250 info!(
4251 module_id = %target_module_id,
4252 connection_id = ctx.connection_id.get(),
4253 "route.bind breaker half-open: admitting one probe"
4254 );
4255 }
4256 guard
4257 }
4258 RouteBindAdmission::Refused {
4259 consecutive_timeouts,
4260 retry_in,
4261 probe_in_flight,
4262 } => {
4263 return Ok(vec![self.route_open_breaker_refusal_frame(
4264 ctx,
4265 &frame,
4266 &target_module_id,
4267 consecutive_timeouts,
4268 retry_in,
4269 probe_in_flight,
4270 )?]);
4271 }
4272 };
4273
4274 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4279 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4280 let pending = match self
4281 .forwarding
4282 .begin_route_bind_relay_for(
4283 ctx.connection_id,
4284 ctx.egress.clone(),
4285 response_version(&frame),
4286 frame.header.corr,
4287 &target_module_id,
4288 principal.clone(),
4289 bound_scope,
4290 Some(project_root),
4291 relay_deadline,
4292 )
4293 .await
4294 {
4295 Ok(pending) => pending,
4296 Err(err) => {
4297 return Ok(vec![self.route_open_refusal_frame(
4298 ctx,
4299 &frame,
4300 &target_module_id,
4301 "relay_reservation_failed",
4302 forwarding_error_code(&err),
4303 err.to_string(),
4304 )?])
4305 }
4306 };
4307 let crate::forwarding::PendingRouteBindRelay {
4308 endpoint,
4309 module_sink,
4310 negotiated_ver,
4311 client_channel,
4312 client_epoch,
4313 module_channel,
4314 module_epoch,
4315 corr: relay_corr,
4316 receiver,
4317 } = pending;
4318 let mut reservation =
4319 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4320
4321 let relay_registration = self
4326 .registry
4327 .get_module_by_connection(endpoint.connection_id)
4328 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4329 let relay_capabilities = relay_registration
4330 .as_ref()
4331 .and_then(|registration| registration.manifest.capabilities.as_ref());
4332 if let Some(stamp) = scope_stamp
4333 .as_ref()
4334 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4335 {
4336 if let Err(refusal) = crate::scopes::check_target_flow_support(
4337 stamp,
4338 &target_module_id,
4339 relay_capabilities,
4340 )
4341 .and_then(|()| {
4342 crate::scopes::check_target_agent_run_support(
4343 stamp,
4344 &target_module_id,
4345 relay_capabilities,
4346 )
4347 }) {
4348 reservation.release_and_disarm();
4349 return Ok(vec![self.route_open_refusal_frame(
4350 ctx,
4351 &frame,
4352 &target_module_id,
4353 refusal.code,
4354 refusal.code,
4355 refusal.message,
4356 )?]);
4357 }
4358 }
4359
4360 let call_mode = if relay_capabilities
4363 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4364 {
4365 Some(derive_call_mode(scope_stamp.as_ref()))
4366 } else {
4367 if let Some(stamp) = scope_stamp.as_mut() {
4368 stamp.expires_at_ms = None;
4369 }
4370 None
4371 };
4372
4373 debug!(
4374 connection_id = ctx.connection_id.get(),
4375 client_channel,
4376 client_epoch,
4377 module_channel,
4378 module_epoch,
4379 "reserved route handle pair"
4380 );
4381 let principal_label = principal_label(&principal);
4384 let relay = ModuleControlRequest::RouteBind {
4385 route_channel: module_channel,
4386 epoch: module_epoch,
4387 target,
4388 identity,
4389 principal: Some(principal),
4390 consumer_capabilities,
4391 role_versions,
4392 admission_facts,
4393 scope: scope_stamp,
4394 call_mode,
4395 };
4396 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4397 RouterError::backend(
4398 0,
4399 frame.header.corr,
4400 format!("failed to encode route.bind request: {err}"),
4401 )
4402 })?;
4403 let relay_frame = Frame::build_with_version(
4404 negotiated_ver,
4405 FrameType::Request,
4406 control_flags(),
4407 0,
4408 0,
4409 relay_corr,
4410 relay_body,
4411 )
4412 .map_err(RouterError::FrameBuild)?;
4413
4414 if let Err(err) = module_sink.send(relay_frame).await {
4415 reservation.release_and_disarm();
4416 return Ok(vec![self.route_open_refusal_frame(
4417 ctx,
4418 &frame,
4419 &target_module_id,
4420 "relay_send_failed",
4421 "target_unavailable",
4422 err.to_string(),
4423 )?]);
4424 }
4425
4426 #[cfg(test)]
4427 if let Some(hold) = &self.route_bind_relay_sent_hold {
4428 hold.wait().await;
4432 hold.wait().await;
4433 }
4434
4435 if self
4440 .forwarding
4441 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4442 .map_err(RouterError::Forwarding)?
4443 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4444 {
4445 self.send_abandoned_route_bind_goodbye(
4446 &module_sink,
4447 negotiated_ver,
4448 module_channel,
4449 module_epoch,
4450 );
4451 }
4452
4453 match timeout_at(relay_deadline, receiver).await {
4454 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4455 reservation.disarm();
4456 if breaker.record_accepted() {
4457 info!(
4458 module_id = %target_module_id,
4459 "route.bind breaker closed: the probe was accepted"
4460 );
4461 }
4462 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4463 Ok(Vec::new())
4464 }
4465 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4466 reservation.release_and_disarm();
4467 breaker.record_inconclusive();
4471 let scope_code = match body.code.as_str() {
4475 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4476 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4477 _ => None,
4478 };
4479 if let Some(code) = scope_code {
4480 self.observe_route_open_refusal(
4481 ctx,
4482 &target_module_id,
4483 "scope_changed_before_commit",
4484 code,
4485 );
4486 return Ok(vec![control_error_body_frame(&frame, body)?]);
4487 }
4488 self.counters
4489 .increment_route_open_refused("module_rejected");
4490 info!(
4491 target: "control",
4492 code = "module_rejected",
4493 module_code = ?body.code,
4494 module_id = ?target_module_id,
4495 connection_id = ctx.connection_id.get(),
4496 "route.open refused"
4497 );
4498 Ok(vec![control_error_body_frame(&frame, body)?])
4499 }
4500 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4501 reservation.release_and_disarm();
4502 breaker.record_inconclusive();
4503 tracing::warn!(
4510 module_id = %target_module_id,
4511 "route.bind relay abandoned: {message}"
4512 );
4513 Ok(vec![self.route_open_refusal_frame(
4514 ctx,
4515 &frame,
4516 &target_module_id,
4517 "relay_abandoned",
4518 "target_unavailable",
4519 message,
4520 )?])
4521 }
4522 Ok(Err(_)) => {
4523 reservation.release_and_disarm();
4524 breaker.record_inconclusive();
4525 Ok(vec![self.route_open_refusal_frame(
4526 ctx,
4527 &frame,
4528 &target_module_id,
4529 "relay_waiter_canceled",
4530 "target_unavailable",
4531 "route.bind relay waiter was canceled before the module responded",
4532 )?])
4533 }
4534 Err(_) => {
4535 reservation.release_and_disarm();
4536 if let Some(opened) = breaker.record_timeout(
4540 self.route_bind_breaker_threshold,
4541 self.route_bind_breaker_cooldown,
4542 ) {
4543 warn!(
4544 module_id = %target_module_id,
4545 consecutive_timeouts = opened.consecutive_timeouts,
4546 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4547 reopened_after_probe = opened.reopened_after_probe,
4548 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4549 );
4550 }
4551 tracing::warn!(
4559 module_id = %target_module_id,
4560 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4561 "route.bind relay timed out: module did not ack within budget"
4562 );
4563 Ok(vec![self.route_open_refusal_frame(
4564 ctx,
4565 &frame,
4566 &target_module_id,
4567 "relay_timed_out",
4568 "module_timeout",
4569 format!(
4570 "module_id '{target_module_id}' did not answer route.bind within {:?}",
4571 route_bind_relay_timeout
4572 ),
4573 )?])
4574 }
4575 }
4576 }
4577
4578 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4579 let response = ClientControlResponse::SupervisorSpawnSnapshot {
4580 snapshot: self.supervisor.spawn_snapshot(),
4581 };
4582 Ok(vec![control_response_body_frame(
4583 &frame,
4584 &response,
4585 "ClientControlResponse::SupervisorSpawnSnapshot",
4586 )?])
4587 }
4588
4589 fn handle_supervisor_spawn_subscribe(
4590 &self,
4591 ctx: &RouteCtx,
4592 frame: Frame,
4593 since: Option<SpawnCursor>,
4594 ) -> Result<Vec<Frame>, RouterError> {
4595 match self.supervisor.subscribe_spawns(
4596 ctx.connection_id,
4597 frame.header.corr,
4598 response_version(&frame),
4599 since,
4600 ctx.egress.clone(),
4601 ) {
4602 Ok(()) => Ok(Vec::new()),
4603 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4604 Ok(vec![control_error_body_frame(
4605 &frame,
4606 ErrorBody {
4607 code: "spawn_cursor_incarnation_mismatch".to_string(),
4608 message: "spawn cursor belongs to a different daemon incarnation"
4609 .to_string(),
4610 detail: Some(serde_json::json!({
4611 "current_daemon_incarnation": current
4612 })),
4613 },
4614 )?])
4615 }
4616 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4617 &frame,
4618 ErrorBody {
4619 code: "spawn_cursor_too_old".to_string(),
4620 message: "spawn cursor predates the retained event ring".to_string(),
4621 detail: Some(serde_json::json!({
4622 "oldest_retained_cursor": oldest
4623 })),
4624 },
4625 )?]),
4626 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4627 0,
4628 frame.header.corr,
4629 format!("failed to open supervisor spawn subscription: {error}"),
4630 )),
4631 }
4632 }
4633
4634 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4635 let generation = self
4636 .registry
4637 .generation()
4638 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4639 let mut modules = Vec::new();
4640 for module in self.supervisor.list() {
4641 let status = module.status_for_control("list").map_err(|err| {
4642 RouterError::backend(
4643 0,
4644 frame.header.corr,
4645 format!("failed to read supervisor status: {err}"),
4646 )
4647 })?;
4648 let (configured, _) = module.configuration().map_err(|err| {
4649 RouterError::backend(
4650 0,
4651 frame.header.corr,
4652 format!("failed to read module configuration: {err}"),
4653 )
4654 })?;
4655 let image = module.running_image_agreement().await;
4657 let resources = Some(module.child_resource_usage());
4660 let pending_reload = Some(reload_verdict(
4661 &configured.program,
4662 status.spawned_from.as_deref(),
4663 image,
4664 ));
4665 modules.push(
4668 SupervisorEntry::new(
4669 status.module_id,
4670 status.state.to_string(),
4671 status.enabled,
4672 status.live,
4673 status.health.status,
4674 )
4675 .with_launch_nonce_env(Some(!cfg!(unix)))
4676 .with_launch_nonce_source(status.launch_nonce_source)
4677 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
4678 .with_protocol(status.protocol)
4679 .with_pending_reload(pending_reload)
4680 .with_last_probe_ms(status.health.last_probe_ms)
4681 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
4682 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
4683 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
4684 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
4685 .with_restart_count(Some(status.restart_count))
4686 .with_max_restarts(Some(status.max_restarts))
4687 .with_lifetime_restarts(Some(status.lifetime_restarts))
4688 .with_spawn_generation(Some(status.spawn_generation))
4689 .with_restart_window_secs(Some(status.restart_window.as_secs()))
4690 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
4691 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
4692 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
4693 .with_resources(resources),
4694 );
4695 }
4696 let response = ClientControlResponse::SupervisorList {
4697 generation,
4698 modules,
4699 };
4700 Ok(vec![control_response_body_frame(
4701 &frame,
4702 &response,
4703 "ClientControlResponse::SupervisorList",
4704 )?])
4705 }
4706
4707 fn handle_supervisor_stderr_tail(
4708 &self,
4709 frame: Frame,
4710 module_id: String,
4711 max_lines: Option<u32>,
4712 max_bytes: Option<u32>,
4713 ) -> Result<Vec<Frame>, RouterError> {
4714 let Some(module) = self.supervisor.get(&module_id) else {
4715 return Ok(vec![control_error_frame(
4716 &frame,
4717 "unknown_module",
4718 format!("module_id '{module_id}' is not supervised"),
4719 )?]);
4720 };
4721
4722 let snapshot = module.stderr_tail(
4723 max_lines.map(|value| value as usize),
4724 max_bytes.map(|value| value as usize),
4725 );
4726
4727 let response = ClientControlResponse::SupervisorStderrTail {
4728 module_id,
4729 tail: StderrTail {
4730 capture: match snapshot.capture {
4731 CaptureState::Captured => StderrCaptureState::Captured,
4732 CaptureState::Incomplete { reason } => {
4733 StderrCaptureState::Incomplete { reason }
4734 }
4735 CaptureState::NotCaptured { reason } => {
4736 StderrCaptureState::NotCaptured { reason }
4737 }
4738 },
4739 entries: snapshot
4740 .entries
4741 .into_iter()
4742 .map(|entry| match entry {
4743 TailEntry::Line {
4744 text,
4745 truncated,
4746 at_ms,
4747 } => StderrTailEntry::Line {
4748 text,
4749 truncated,
4750 at_ms,
4751 },
4752 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
4753 })
4754 .collect(),
4755 dropped_lines: snapshot.dropped_lines,
4756 },
4757 };
4758 Ok(vec![control_response_body_frame(
4759 &frame,
4760 &response,
4761 "ClientControlResponse::SupervisorStderrTail",
4762 )?])
4763 }
4764
4765 async fn handle_supervisor_terminals(
4766 &self,
4767 frame: Frame,
4768 module_id: String,
4769 ) -> Result<Vec<Frame>, RouterError> {
4770 let Some(module) = self.supervisor.get(&module_id) else {
4771 return Ok(vec![control_error_frame(
4772 &frame,
4773 "unknown_module",
4774 format!("module_id '{module_id}' is not supervised"),
4775 )?]);
4776 };
4777
4778 let terminals = module
4781 .read_durable_terminal_history()
4782 .await
4783 .map_err(|error| {
4784 RouterError::backend(
4785 0,
4786 frame.header.corr,
4787 format!("failed to read terminal history: {error}"),
4788 )
4789 })?;
4790 let response = ClientControlResponse::SupervisorTerminals {
4791 module_id,
4792 terminals,
4793 };
4794 Ok(vec![control_response_body_frame(
4795 &frame,
4796 &response,
4797 "ClientControlResponse::SupervisorTerminals",
4798 )?])
4799 }
4800
4801 fn handle_supervisor_routes(
4802 &self,
4803 frame: Frame,
4804 module_id: Option<String>,
4805 ) -> Result<Vec<Frame>, RouterError> {
4806 let modules = self
4807 .forwarding
4808 .route_census(module_id.as_deref())
4809 .map_err(RouterError::Forwarding)?
4810 .into_iter()
4811 .map(|(module_id, routes)| SupervisorRouteModule {
4812 module_id,
4813 routes: routes
4814 .into_iter()
4815 .map(|route| SupervisorRoute {
4816 consumer: match route.principal {
4817 Principal::Reserved { module_id } => {
4818 SupervisorRouteConsumer::Reserved { module_id }
4819 }
4820 Principal::Direct | Principal::Unverified => {
4821 SupervisorRouteConsumer::Direct {
4822 connection_id: route.goodbye_target.connection_id.get(),
4823 }
4824 }
4825 },
4826 age_ms: Instant::now()
4827 .saturating_duration_since(route.bound_at)
4828 .as_millis()
4829 .try_into()
4830 .unwrap_or(u64::MAX),
4831 draining: route.draining,
4832 drain_reason: route.drain_reason,
4833 })
4834 .collect(),
4835 })
4836 .collect();
4837 let response = ClientControlResponse::SupervisorRoutes { modules };
4838 Ok(vec![control_response_body_frame(
4839 &frame,
4840 &response,
4841 "ClientControlResponse::SupervisorRoutes",
4842 )?])
4843 }
4844
4845 async fn handle_supervisor_provenance(
4846 &self,
4847 frame: Frame,
4848 module_id: Option<String>,
4849 ) -> Result<Vec<Frame>, RouterError> {
4850 let mut selected = if let Some(module_id) = module_id {
4851 let Some(module) = self.supervisor.get(&module_id) else {
4852 return Ok(vec![control_error_frame(
4853 &frame,
4854 "unknown_module",
4855 format!("module_id '{module_id}' is not supervised"),
4856 )?]);
4857 };
4858 vec![module]
4859 } else {
4860 self.supervisor.list()
4861 };
4862
4863 let mut modules = Vec::with_capacity(selected.len());
4864 for module in selected.drain(..) {
4865 let (status, observed_image) = module
4866 .status_and_running_image_agreement()
4867 .await
4868 .map_err(|err| {
4869 RouterError::backend(
4870 0,
4871 frame.header.corr,
4872 format!("failed to read supervisor status: {err}"),
4873 )
4874 })?;
4875 let module_declared = self
4876 .registry
4877 .get_module(&status.module_id)
4878 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4879 .and_then(|registration| registration.manifest.provenance)
4880 .map(|build| ModuleDeclaredProvenance::Reported { build })
4881 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4882 #[cfg(test)]
4883 let running_image = match &self.provenance_probe_override {
4884 Some(result) => result.clone(),
4885 None => observed_image,
4886 };
4887 #[cfg(not(test))]
4888 let running_image = observed_image;
4889 modules.push(SupervisorModuleProvenance {
4890 module_id: status.module_id,
4891 module_declared,
4892 daemon_observed: SupervisorObservedProcess {
4893 launch_nonce_source: status.launch_nonce_source,
4894 pid: status.pid,
4895 spawned_at_ms: status.spawned_at_ms,
4896 spawned_from: status.spawned_from,
4897 running_image,
4898 },
4899 });
4900 }
4901 let daemon = SupervisorDaemonProvenance {
4902 daemon_build: self.daemon_provenance.build.clone(),
4903 daemon_observed: DaemonObservedProcess {
4904 pid: self.daemon_provenance.pid,
4905 started_at_ms: self
4906 .daemon_provenance
4907 .start_clock
4908 .map(|clock| clock.started_at_ms())
4909 .or(self.daemon_provenance.started_at_ms),
4910 running_image: self
4911 .daemon_provenance
4912 .probe
4913 .observe(
4914 self.daemon_provenance.pid,
4915 self.daemon_provenance.executable_path.as_deref(),
4916 self.daemon_provenance.executable_identity.clone(),
4917 self.daemon_provenance.process_start_time,
4918 )
4919 .await,
4920 },
4921 };
4922 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
4923 Ok(vec![control_response_body_frame(
4924 &frame,
4925 &response,
4926 "ClientControlResponse::SupervisorProvenance",
4927 )?])
4928 }
4929
4930 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4931 self.refresh_capability_requirements();
4932 let generation = self
4933 .registry
4934 .generation()
4935 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4936 let modules = self
4937 .supervisor
4938 .list()
4939 .into_iter()
4940 .map(|module| {
4941 let status = module.status_for_control("health").map_err(|err| {
4942 RouterError::backend(
4943 0,
4944 frame.header.corr,
4945 format!("failed to read supervisor health: {err}"),
4946 )
4947 })?;
4948 let module_id = status.module_id;
4949 let capability_detail = self
4950 .capability_evaluator
4951 .required_problem_detail(&module_id);
4952 Ok(SupervisorHealthEntry {
4953 module_id,
4954 status: status.health.status,
4955 detail: append_capability_problem_detail(
4956 status.health.detail,
4957 capability_detail,
4958 ),
4959 metrics: status.health.metrics,
4960 consecutive_failures: status.health.consecutive_failures,
4961 late_answer_count: status.health.late_answer_count,
4962 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
4963 last_action: status.health.last_action,
4964 last_action_ms: status.health.last_action_ms,
4965 last_probe_ms: status.health.last_probe_ms,
4966 })
4967 })
4968 .collect::<Result<Vec<_>, RouterError>>()?;
4969 let response = ClientControlResponse::SupervisorHealth {
4970 generation,
4971 modules,
4972 };
4973 Ok(vec![control_response_body_frame(
4974 &frame,
4975 &response,
4976 "ClientControlResponse::SupervisorHealth",
4977 )?])
4978 }
4979
4980 async fn handle_supervisor_restart(
4981 &self,
4982 frame: Frame,
4983 module_id: String,
4984 drain_timeout_ms: Option<u64>,
4985 ) -> Result<Vec<Frame>, RouterError> {
4986 let operation_lock = self.supervisor.operation_lock();
4987 let _operation_guard = operation_lock.lock().await;
4988 let Some(module) = self.supervisor.get(&module_id) else {
4989 return Ok(vec![control_error_frame(
4990 &frame,
4991 "unknown_module",
4992 format!("module_id '{module_id}' is not supervised"),
4993 )?]);
4994 };
4995
4996 self.route_outages.mark_operator_action(&module_id);
4997 if let Err(err) = module.restart(drain_timeout_ms).await {
4998 self.route_outages
4999 .operator_action_ended_unrefused(&module_id);
5000 let (code, message) = match err {
5001 crate::supervise::SuperviseError::Disabled { .. } => {
5002 ("module_disabled", err.to_string())
5003 }
5004 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5005 ("swap_in_progress", err.to_string())
5006 }
5007 _ => (
5008 "target_unavailable",
5009 format!("failed to restart module_id '{module_id}': {err}"),
5010 ),
5011 };
5012 return Ok(vec![control_error_frame(&frame, code, message)?]);
5013 }
5014
5015 let response = ClientControlResponse::SupervisorAck {
5016 module_id,
5017 applied: true,
5018 };
5019 Ok(vec![control_response_body_frame(
5020 &frame,
5021 &response,
5022 "ClientControlResponse::SupervisorAck",
5023 )?])
5024 }
5025
5026 async fn handle_supervisor_swap(
5030 &self,
5031 frame: Frame,
5032 module_id: String,
5033 ready_timeout_ms: Option<u64>,
5034 ) -> Result<Vec<Frame>, RouterError> {
5035 let module = {
5042 let operation_lock = self.supervisor.operation_lock();
5043 let _operation_guard = operation_lock.lock().await;
5044 self.supervisor.get(&module_id)
5045 };
5046 let Some(module) = module else {
5047 return Ok(vec![control_error_frame(
5048 &frame,
5049 "unknown_module",
5050 format!("module_id '{module_id}' is not supervised"),
5051 )?]);
5052 };
5053
5054 self.route_outages.mark_operator_action(&module_id);
5055 if let Err(err) = module
5056 .swap(ready_timeout_ms.map(Duration::from_millis))
5057 .await
5058 {
5059 self.route_outages
5060 .operator_action_ended_unrefused(&module_id);
5061 use crate::supervise::SuperviseError;
5062 let message = err.to_string();
5063 let error = match err {
5064 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5065 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5066 code: "swap_refused".to_string(),
5067 message,
5068 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5069 },
5070 SuperviseError::SwapFailed {
5071 arm,
5072 candidate_exit,
5073 ..
5074 } => ErrorBody {
5075 code: "swap_failed".to_string(),
5076 message,
5077 detail: Some(serde_json::json!({
5078 "arm": arm.as_str(),
5079 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5080 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5081 })),
5082 },
5083 _ => ErrorBody::new(
5084 "target_unavailable",
5085 format!("failed to swap module_id '{module_id}': {message}"),
5086 ),
5087 };
5088 return Ok(vec![control_error_body_frame(&frame, error)?]);
5089 }
5090 self.route_outages
5094 .operator_action_ended_unrefused(&module_id);
5095
5096 let response = ClientControlResponse::SupervisorAck {
5097 module_id,
5098 applied: true,
5099 };
5100 Ok(vec![control_response_body_frame(
5101 &frame,
5102 &response,
5103 "ClientControlResponse::SupervisorAck",
5104 )?])
5105 }
5106
5107 async fn handle_supervisor_reload(
5108 &self,
5109 frame: Frame,
5110 module_id: String,
5111 ) -> Result<Vec<Frame>, RouterError> {
5112 let operation_lock = self.supervisor.operation_lock();
5113 let _operation_guard = operation_lock.lock().await;
5114 let Some(module) = self.supervisor.get(&module_id) else {
5115 return Ok(vec![control_error_frame(
5116 &frame,
5117 "unknown_module",
5118 format!("module_id '{module_id}' is not supervised"),
5119 )?]);
5120 };
5121
5122 self.route_outages.mark_operator_action(&module_id);
5123 if let Err(err) = module.reload().await {
5124 self.route_outages
5125 .operator_action_ended_unrefused(&module_id);
5126 let (code, message) = match err {
5127 crate::supervise::SuperviseError::Disabled { .. } => {
5128 ("module_disabled", err.to_string())
5129 }
5130 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5131 ("swap_in_progress", err.to_string())
5132 }
5133 _ => (
5134 "reload_failed",
5135 format!("failed to reload module_id '{module_id}': {err}"),
5136 ),
5137 };
5138 return Ok(vec![control_error_frame(&frame, code, message)?]);
5139 }
5140
5141 let response = ClientControlResponse::SupervisorAck {
5142 module_id,
5143 applied: true,
5144 };
5145 Ok(vec![control_response_body_frame(
5146 &frame,
5147 &response,
5148 "ClientControlResponse::SupervisorAck",
5149 )?])
5150 }
5151
5152 async fn handle_supervisor_rescan(
5153 &self,
5154 frame: Frame,
5155 preview: bool,
5156 ) -> Result<Vec<Frame>, RouterError> {
5157 let Some(context) = self.rescan.clone() else {
5158 return Ok(vec![control_error_frame(
5159 &frame,
5160 "rescan_unavailable",
5161 "the daemon was not started with a reloadable config path".to_string(),
5162 )?]);
5163 };
5164
5165 let operation_lock = self.supervisor.operation_lock();
5166 let _operation_guard = operation_lock.lock().await;
5167 let loaded = match crate::daemon_config::load_with_activity(&context.config_path) {
5168 Ok(config) => config,
5169 Err(err) => {
5170 return Ok(vec![control_error_frame(
5171 &frame,
5172 "invalid_daemon_config",
5173 format!("supervisor rescan rejected daemon config: {err}"),
5174 )?])
5175 }
5176 };
5177 let Some((config, activity_config)) = loaded else {
5185 return Ok(vec![control_error_frame(
5186 &frame,
5187 "invalid_daemon_config",
5188 format!(
5189 "daemon config not found at {}; refusing to rescan (an absent config would \
5190 retire every supervised module)",
5191 context.config_path.display()
5192 ),
5193 )?]);
5194 };
5195 let (
5196 configured_port,
5197 storage_config,
5198 admission_facts_carrier_module_id,
5199 admission_facts_targets,
5200 scope_authority_owners,
5201 modules,
5202 reserved_capabilities,
5203 ) = (
5204 config.port,
5205 config.storage,
5206 config.admission_facts_carrier_module_id,
5207 config.admission_facts_targets,
5208 config.scope_authority_owners,
5209 config.modules,
5210 config.reserved_capabilities,
5211 );
5212
5213 let mut restart_required = Vec::new();
5222 for section in RestartRequiredSection::ALL {
5223 let changed = match section {
5224 RestartRequiredSection::Port => configured_port != context.configured_port,
5225 RestartRequiredSection::Storage => storage_config != context.storage_config,
5226 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5227 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5228 }
5229 RestartRequiredSection::AdmissionFactsTargets => {
5230 admission_facts_targets != context.admission_facts_targets
5231 }
5232 RestartRequiredSection::ScopeAuthorityOwners => {
5233 scope_authority_owners != context.scope_authority_owners
5234 }
5235 };
5236 if changed {
5237 restart_required.push(section.label().to_string());
5238 }
5239 }
5240 if !restart_required.is_empty() {
5241 warn!(
5242 config_path = %context.config_path.display(),
5243 sections = %restart_required.join(", "),
5244 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5245 );
5246 }
5247
5248 for configured in &modules {
5249 if let Err(err) = validate_spec(&configured.module_spec()) {
5250 return Ok(vec![control_error_frame(
5251 &frame,
5252 "invalid_daemon_config",
5253 format!("supervisor rescan rejected daemon config: {err}"),
5254 )?]);
5255 }
5256 }
5257
5258 let configured_capabilities = modules
5259 .iter()
5260 .map(|module| (module.module_id.clone(), module.enabled))
5261 .collect::<Vec<_>>();
5262 let preview_capability_warnings = if preview {
5263 let (_, registrations) = self.runtime_capability_snapshot()?;
5264 let current_modules = self
5265 .supervisor
5266 .list()
5267 .into_iter()
5268 .map(|module| module.module_id().to_string())
5269 .collect::<BTreeSet<_>>();
5270 let resulting_modules = configured_capabilities.clone();
5271 let removed = current_modules
5272 .into_iter()
5273 .filter(|module_id| {
5274 !resulting_modules
5275 .iter()
5276 .any(|(configured_id, _)| configured_id == module_id)
5277 })
5278 .collect::<Vec<_>>();
5279 self.capability_evaluator.preview_removal_warnings(
5280 resulting_modules,
5281 &removed,
5282 ®istrations,
5283 )
5284 } else {
5285 Vec::new()
5286 };
5287 let result = match self
5288 .reconcile_supervised_modules(&context.supervisor, modules, preview)
5289 .await
5290 {
5291 Ok(result) => result,
5292 Err(message) => {
5293 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5294 }
5295 };
5296 if !preview {
5297 if let Err(code) = self.configure_activity(activity_config).await {
5298 warn!(
5299 code,
5300 "activity retention could not be applied during rescan"
5301 );
5302 }
5303 self.capability_evaluator
5304 .configure(configured_capabilities, reserved_capabilities);
5305 self.capability_evaluator.wake_deadline_loop();
5306 self.refresh_capability_requirements();
5307 }
5308 let mut result = result;
5309 result.restart_required = restart_required;
5310 result.capability_warnings = preview_capability_warnings;
5311 let response = ClientControlResponse::SupervisorRescan { result };
5312 Ok(vec![control_response_body_frame(
5313 &frame,
5314 &response,
5315 "ClientControlResponse::SupervisorRescan",
5316 )?])
5317 }
5318
5319 async fn handle_supervisor_release_reserved(
5320 &self,
5321 frame: Frame,
5322 module_id: String,
5323 ) -> Result<Vec<Frame>, RouterError> {
5324 let Some(context) = self.rescan.clone() else {
5325 return Ok(vec![control_error_frame(
5326 &frame,
5327 "release_unavailable",
5328 "reserved-id release requires a daemon started with a reloadable config path",
5329 )?]);
5330 };
5331 let operation_lock = self.supervisor.operation_lock();
5332 let _operation_guard = operation_lock.lock().await;
5333 let loaded = match crate::daemon_config::load(&context.config_path) {
5334 Ok(Some(config)) => config,
5335 Ok(None) => {
5336 return Ok(vec![control_error_frame(
5337 &frame,
5338 "invalid_daemon_config",
5339 format!(
5340 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5341 context.config_path.display()
5342 ),
5343 )?])
5344 }
5345 Err(err) => {
5346 return Ok(vec![control_error_frame(
5347 &frame,
5348 "invalid_daemon_config",
5349 format!("unable to verify reserved-id release against daemon config: {err}"),
5350 )?])
5351 }
5352 };
5353 if loaded
5354 .modules
5355 .iter()
5356 .any(|configured| configured.module_id == module_id)
5357 {
5358 return Ok(vec![control_error_frame(
5359 &frame,
5360 "reserved_module_configured",
5361 format!(
5362 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5363 ),
5364 )?]);
5365 }
5366 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5367 return Ok(vec![control_error_frame(
5368 &frame,
5369 "reserved_gate_not_retained",
5370 format!(
5371 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5372 ),
5373 )?]);
5374 }
5375
5376 let response = ClientControlResponse::SupervisorAck {
5377 module_id,
5378 applied: true,
5379 };
5380 Ok(vec![control_response_body_frame(
5381 &frame,
5382 &response,
5383 "ClientControlResponse::SupervisorAck",
5384 )?])
5385 }
5386
5387 async fn reconcile_supervised_modules(
5396 &self,
5397 supervisor: &Supervisor,
5398 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5399 preview: bool,
5400 ) -> Result<SupervisorRescanResult, String> {
5401 let mut current = BTreeMap::new();
5402 for module in self.supervisor.list() {
5403 let (spec, health) = module.configuration().map_err(|err| {
5404 format!(
5405 "failed to read configuration for module_id '{}': {err}",
5406 module.module_id()
5407 )
5408 })?;
5409 let enabled = module
5410 .status()
5411 .map_err(|err| {
5412 format!(
5413 "failed to read status for module_id '{}': {err}",
5414 module.module_id()
5415 )
5416 })?
5417 .enabled;
5418 current.insert(
5419 module.module_id().to_string(),
5420 (module, spec, health, enabled),
5421 );
5422 }
5423 let configured = configured_modules
5424 .into_iter()
5425 .map(|module| (module.module_id.clone(), module))
5426 .collect::<BTreeMap<_, _>>();
5427
5428 let added = configured
5429 .keys()
5430 .filter(|module_id| !current.contains_key(*module_id))
5431 .cloned()
5432 .collect::<Vec<_>>();
5433 let removed = current
5434 .keys()
5435 .filter(|module_id| !configured.contains_key(*module_id))
5436 .cloned()
5437 .collect::<Vec<_>>();
5438 let mut changed_pending_reload = Vec::new();
5439 let mut configuration_changes = BTreeSet::new();
5440 let mut enabled_changes = BTreeSet::new();
5441 let mut unchanged = 0_u32;
5442
5443 for (module_id, configured_module) in &configured {
5444 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5445 else {
5446 continue;
5447 };
5448 let launch_changed = *current_spec != configured_module.module_spec();
5452 let configuration_changed =
5453 launch_changed || *current_health != configured_module.health;
5454 let enabled_changed = *current_enabled != configured_module.enabled;
5455 if configuration_changed {
5456 configuration_changes.insert(module_id.clone());
5457 }
5458 if launch_changed {
5459 changed_pending_reload.push(module_id.clone());
5460 }
5461 if enabled_changed {
5462 enabled_changes.insert(module_id.clone());
5463 }
5464 if !configuration_changed && !enabled_changed {
5465 unchanged = unchanged.saturating_add(1);
5466 }
5467 }
5468
5469 if preview {
5475 return Ok(SupervisorRescanResult {
5476 added,
5477 removed,
5478 changed_pending_reload,
5479 enabled_changes: enabled_changes.iter().cloned().collect(),
5480 unchanged,
5481 preview: true,
5482 restart_required: Vec::new(),
5486 capability_warnings: Vec::new(),
5487 });
5488 }
5489
5490 for module_id in &removed {
5491 let module = ¤t
5492 .get(module_id)
5493 .expect("removed module came from current supervisor state")
5494 .0;
5495 module.retire().await.map_err(|err| {
5496 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5497 })?;
5498 self.supervisor.record_rescan_removal(module_id);
5532 self.supervisor.retire(module_id);
5533 self.route_outages.forget(module_id);
5534 }
5535
5536 for module_id in configured.keys() {
5537 let Some((module, _, _, _)) = current.get(module_id) else {
5538 continue;
5539 };
5540 let configured_module = configured
5541 .get(module_id)
5542 .expect("configured module id came from configured map");
5543 if configuration_changes.contains(module_id) {
5544 module
5545 .update_configuration(
5546 configured_module.module_spec(),
5547 configured_module.health.clone(),
5548 configured_module.drain_timeout_ms,
5549 )
5550 .await
5551 .map_err(|err| {
5552 format!(
5553 "failed to update module_id '{module_id}' configuration during rescan: {err}"
5554 )
5555 })?;
5556 }
5557 if enabled_changes.contains(module_id) {
5558 self.route_outages.mark_operator_action(module_id);
5561 module
5562 .set_enabled(configured_module.enabled)
5563 .await
5564 .map_err(|err| {
5565 self.route_outages.operator_action_ended_unrefused(module_id);
5566 format!(
5567 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5568 configured_module.enabled
5569 )
5570 })?;
5571 }
5572 }
5573
5574 for module_id in &added {
5575 let configured_module = configured
5576 .get(module_id)
5577 .expect("added module id came from configured map");
5578 supervisor
5579 .supervise_configured_with_health(
5580 configured_module.module_spec(),
5581 configured_module.enabled,
5582 configured_module.health.clone(),
5583 configured_module.drain_timeout_ms,
5584 configured_module.restart,
5585 )
5586 .map_err(|err| {
5587 format!("failed to add module_id '{module_id}' during rescan: {err}")
5588 })?;
5589 }
5590
5591 Ok(SupervisorRescanResult {
5592 added,
5593 removed,
5594 changed_pending_reload,
5595 enabled_changes: enabled_changes.iter().cloned().collect(),
5596 unchanged,
5597 preview: false,
5598 restart_required: Vec::new(),
5601 capability_warnings: Vec::new(),
5602 })
5603 }
5604
5605 async fn handle_supervisor_set_enabled(
5606 &self,
5607 frame: Frame,
5608 module_id: String,
5609 enabled: bool,
5610 ) -> Result<Vec<Frame>, RouterError> {
5611 let operation_lock = self.supervisor.operation_lock();
5612 let _operation_guard = operation_lock.lock().await;
5613 let Some(module) = self.supervisor.get(&module_id) else {
5614 return Ok(vec![control_error_frame(
5615 &frame,
5616 "unknown_module",
5617 format!("module_id '{module_id}' is not supervised"),
5618 )?]);
5619 };
5620
5621 self.route_outages.mark_operator_action(&module_id);
5624 let applied = match module.set_enabled(enabled).await {
5625 Ok(applied) => applied,
5626 Err(err) => {
5627 self.route_outages
5628 .operator_action_ended_unrefused(&module_id);
5629 return Ok(vec![control_error_frame(
5630 &frame,
5631 "target_unavailable",
5632 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
5633 )?]);
5634 }
5635 };
5636 if !applied {
5637 self.route_outages
5640 .operator_action_ended_unrefused(&module_id);
5641 }
5642
5643 self.capability_evaluator.wake_deadline_loop();
5644 self.refresh_capability_requirements();
5645 let response = ClientControlResponse::SupervisorAck { module_id, applied };
5646 Ok(vec![control_response_body_frame(
5647 &frame,
5648 &response,
5649 "ClientControlResponse::SupervisorAck",
5650 )?])
5651 }
5652
5653 async fn handle_supervisor_health_probe(
5654 &self,
5655 frame: Frame,
5656 module_id: String,
5657 ) -> Result<Vec<Frame>, RouterError> {
5658 self.refresh_capability_requirements();
5659 let Some(registration) = self
5660 .registry
5661 .get_module(&module_id)
5662 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5663 else {
5664 return Ok(vec![control_error_frame(
5665 &frame,
5666 "unknown_module",
5667 format!("module_id '{module_id}' is not registered"),
5668 )?]);
5669 };
5670
5671 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
5684 {
5685 return Ok(vec![control_error_frame(
5686 &frame,
5687 "health_not_advertised",
5688 format!("module_id '{module_id}' did not advertise health.check"),
5689 )?]);
5690 }
5691
5692 let deadline = Instant::now() + self.health_probe_timeout;
5693 let pending = match self.forwarding.begin_module_control_rpc_for(
5694 &module_id,
5695 MODULE_CONTROL_OP_HEALTH_CHECK,
5696 deadline,
5697 ) {
5698 Ok(pending) => pending,
5699 Err(err) => {
5700 return Ok(vec![control_error_frame(
5701 &frame,
5702 forwarding_error_code(&err),
5703 err.to_string(),
5704 )?])
5705 }
5706 };
5707
5708 let PendingModuleControlRpc {
5709 endpoint,
5710 module_sink,
5711 negotiated_ver,
5712 corr: probe_corr,
5713 receiver,
5714 } = pending;
5715 let mut guard =
5716 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
5717 let probe_body =
5718 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5719 RouterError::backend(
5720 0,
5721 frame.header.corr,
5722 format!("failed to encode health.check request: {err}"),
5723 )
5724 })?;
5725 let probe_frame = Frame::build_with_version(
5726 negotiated_ver,
5727 FrameType::Request,
5728 control_flags(),
5729 0,
5730 0,
5731 probe_corr,
5732 probe_body,
5733 )
5734 .map_err(RouterError::FrameBuild)?;
5735
5736 if let Err(err) = module_sink.send(probe_frame).await {
5737 return Ok(vec![control_error_frame(
5738 &frame,
5739 "target_unavailable",
5740 err.to_string(),
5741 )?]);
5742 }
5743
5744 match timeout_at(deadline, receiver).await {
5745 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5746 guard.disarm();
5747 let Some(report) = response.health_report() else {
5748 return Ok(vec![control_error_frame(
5749 &frame,
5750 "invalid_control_body",
5751 "health.check RPC returned a non-health response",
5752 )?]);
5753 };
5754 let HealthReport {
5759 status,
5760 detail,
5761 metrics,
5762 } = report;
5763 let capability_detail = self
5764 .capability_evaluator
5765 .required_problem_detail(&module_id);
5766 let response = ClientControlResponse::SupervisorHealthProbe {
5767 module_id,
5768 status,
5769 detail: append_capability_problem_detail(detail, capability_detail),
5770 metrics,
5771 };
5772 Ok(vec![control_response_body_frame(
5773 &frame,
5774 &response,
5775 "ClientControlResponse::SupervisorHealthProbe",
5776 )?])
5777 }
5778 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
5779 guard.disarm();
5780 Ok(vec![control_error_body_frame(&frame, body)?])
5781 }
5782 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5783 guard.disarm();
5784 Ok(vec![control_error_frame(
5785 &frame,
5786 "target_unavailable",
5787 message,
5788 )?])
5789 }
5790 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5791 guard.disarm();
5792 Ok(vec![control_error_frame(
5793 &frame,
5794 "invalid_control_body",
5795 message,
5796 )?])
5797 }
5798 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5799 guard.disarm();
5800 Ok(vec![control_error_frame(
5801 &frame,
5802 "invalid_control_body",
5803 format!("expected module-control op '{expected}', got '{actual}'"),
5804 )?])
5805 }
5806 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
5807 guard.disarm();
5808 Ok(vec![control_error_frame(
5809 &frame,
5810 "module_timeout",
5811 format!(
5812 "module_id '{module_id}' answered health.check after {:?}",
5813 self.health_probe_timeout
5814 ),
5815 )?])
5816 }
5817 Ok(Err(_)) => Ok(vec![control_error_frame(
5818 &frame,
5819 "target_unavailable",
5820 "health.check waiter was canceled before the module responded",
5821 )?]),
5822 Err(_) => Ok(vec![control_error_frame(
5823 &frame,
5824 "module_timeout",
5825 format!(
5826 "module_id '{module_id}' did not answer health.check within {:?}",
5827 self.health_probe_timeout
5828 ),
5829 )?]),
5830 }
5831 }
5832
5833 fn supervisor_status(
5834 &self,
5835 module_id: &str,
5836 corr: u64,
5837 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5838 self.supervisor
5839 .get(module_id)
5840 .map(|module| {
5841 let warming = module.is_warming_for_control("status").map_err(|err| {
5842 RouterError::backend(
5843 0,
5844 corr,
5845 format!(
5846 "failed to read supervisor warming state for module_id '{module_id}': {err}"
5847 ),
5848 )
5849 })?;
5850 module.status_for_control("status").map_err(|err| {
5851 RouterError::backend(
5852 0,
5853 corr,
5854 format!(
5855 "failed to read supervisor status for module_id '{module_id}': {err}"
5856 ),
5857 )
5858 }).map(|status| (status, warming))
5859 })
5860 .transpose()
5861 }
5862
5863 fn guard_module_control_op(
5864 &self,
5865 frame: &Frame,
5866 module_id: &str,
5867 op: &str,
5868 ) -> Result<Option<Frame>, RouterError> {
5869 if self.module_grants_op(module_id, op, frame.header.corr)? {
5870 return Ok(None);
5871 }
5872
5873 Ok(Some(control_error_frame(
5874 frame,
5875 "op_not_allowed",
5876 format!("module_id '{module_id}' did not grant control op '{op}'"),
5877 )?))
5878 }
5879
5880 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
5881 let Some(registration) = self
5882 .registry
5883 .get_module(module_id)
5884 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
5885 else {
5886 return Ok(false);
5887 };
5888 Ok(module_registration_grants_op(®istration.control_ops, op))
5889 }
5890
5891 fn handle_status_update(
5892 &self,
5893 endpoint: ModuleEndpointId,
5894 frame: Frame,
5895 ) -> Result<Vec<Frame>, RouterError> {
5896 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
5897 Ok(update) => update,
5898 Err(err) => {
5899 if is_known_module_push_op(&frame.body) {
5904 return Ok(vec![control_error_frame(
5905 &frame,
5906 "invalid_control_body",
5907 format!("malformed module control push body: {err}"),
5908 )?]);
5909 }
5910 return Ok(Vec::new());
5911 }
5912 };
5913
5914 match update {
5915 ModuleControlPush::RouteStatus {
5916 route_channel,
5917 route_epoch,
5918 status,
5919 } => {
5920 self.forwarding
5921 .cache_status(endpoint, route_channel, route_epoch, status)
5922 .map_err(RouterError::Forwarding)?;
5923 }
5924 }
5925 Ok(Vec::new())
5926 }
5927
5928 fn handle_route_poll(
5929 &self,
5930 ctx: &RouteCtx,
5931 frame: Frame,
5932 route_channel: u16,
5933 route_epoch: u32,
5934 kind: PollKind,
5935 ) -> Result<Vec<Frame>, RouterError> {
5936 let snapshot = self
5937 .forwarding
5938 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
5939 .map_err(RouterError::Forwarding)?;
5940 let response = match (kind, snapshot) {
5941 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
5942 ClientControlResponse::RoutePoll {
5943 route_channel,
5944 route_epoch,
5945 status,
5946 live: None,
5947 }
5948 }
5949 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5950 route_channel,
5951 route_epoch,
5952 status: None,
5953 live: None,
5954 },
5955 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
5956 let live = self
5982 .process_liveness
5983 .as_ref()
5984 .and_then(|source| source.process_live(&module_id))
5985 .unwrap_or(true);
5986 ClientControlResponse::RoutePoll {
5987 route_channel,
5988 route_epoch,
5989 status: None,
5990 live: Some(live),
5991 }
5992 }
5993 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
5994 route_channel,
5995 route_epoch,
5996 status: None,
5997 live: Some(false),
5998 },
5999 };
6000
6001 Ok(vec![control_response_body_frame(
6002 &frame,
6003 &response,
6004 "ClientControlResponse::RoutePoll",
6005 )?])
6006 }
6007
6008 pub(crate) fn observe_module_control_completion(
6009 &self,
6010 completion: ModuleControlRpcCompletion,
6011 ) -> bool {
6012 match completion {
6013 ModuleControlRpcCompletion::Unknown => false,
6014 ModuleControlRpcCompletion::Settled => true,
6015 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6016 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6017 info!(
6018 module_id = %module_id,
6019 latency_ms,
6020 "late health.check answer proves the module is alive"
6021 );
6022 match self
6023 .supervisor
6024 .record_late_health_answer(&module_id, latency_ms)
6025 {
6026 Ok(true) => {}
6027 Ok(false) => debug!(
6028 module_id = %module_id,
6029 latency_ms,
6030 "late health.check answer has no active supervisor snapshot"
6031 ),
6032 Err(err) => warn!(
6033 module_id = %module_id,
6034 latency_ms,
6035 error = %err,
6036 "failed to record late health.check answer"
6037 ),
6038 }
6039 true
6040 }
6041 }
6042 }
6043
6044 fn refuse_to_end_module_connection_for_a_client(
6065 &self,
6066 module_connection_id: ConnectionId,
6067 corr: u64,
6068 err: ForwardingError,
6069 ) -> Result<(), RouterError> {
6070 if let ForwardingError::ConnectionClosing { connection_id } = err {
6071 if connection_id != module_connection_id {
6072 warn!(
6073 module_connection_id = module_connection_id.get(),
6074 client_connection_id = connection_id.get(),
6075 corr,
6076 "dropping a route.bind response for a closing client; the module connection keeps serving"
6077 );
6078 return Ok(());
6079 }
6080 }
6081 Err(RouterError::Forwarding(err))
6082 }
6083
6084 fn handle_module_relay_response(
6085 &self,
6086 connection_id: ConnectionId,
6087 frame: Frame,
6088 ) -> Result<Vec<Frame>, RouterError> {
6089 let mut secondary_error = None;
6090 let outcome = match frame.header.ty {
6091 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6092 Ok(probe) if probe.op == "route.bind" => {
6093 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6094 Ok(ModuleControlResponse::RouteBindAck {}) => {
6095 RouteBindRelayOutcome::Accepted
6096 }
6097 Ok(other) => {
6098 let message =
6099 format!("route.bind response carried unexpected body: {other:?}");
6100 secondary_error = Some(control_error_frame(
6101 &frame,
6102 "invalid_control_body",
6103 message.clone(),
6104 )?);
6105 RouteBindRelayOutcome::ModuleGone(message)
6106 }
6107 Err(err) => {
6108 let message = format!("malformed route.bind response body: {err}");
6109 secondary_error = Some(control_error_frame(
6110 &frame,
6111 "invalid_control_body",
6112 message.clone(),
6113 )?);
6114 RouteBindRelayOutcome::ModuleGone(message)
6115 }
6116 }
6117 }
6118 Ok(probe) => {
6119 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6120 {
6121 Ok(response) => ModuleControlRpcOutcome::Response(response),
6122 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6123 "malformed {} response body: {err}",
6124 probe.op
6125 )),
6126 };
6127 let completion = self
6128 .forwarding
6129 .complete_module_control_rpc(
6130 connection_id,
6131 frame.header.corr,
6132 Some(&probe.op),
6133 outcome,
6134 )
6135 .map_err(RouterError::Forwarding)?;
6136 if !self.observe_module_control_completion(completion) {
6137 debug!(
6138 connection_id = connection_id.get(),
6139 corr = frame.header.corr,
6140 op = %probe.op,
6141 "dropping late or unknown module-control RPC response"
6142 );
6143 }
6144 return Ok(Vec::new());
6145 }
6146 Err(err) => {
6147 if let Some(expected_op) = self
6148 .forwarding
6149 .pending_module_control_op(connection_id, frame.header.corr)
6150 .map_err(RouterError::Forwarding)?
6151 {
6152 let completion = self
6153 .forwarding
6154 .complete_module_control_rpc(
6155 connection_id,
6156 frame.header.corr,
6157 None,
6158 ModuleControlRpcOutcome::MalformedResponse(format!(
6159 "malformed {expected_op} response body: {err}"
6160 )),
6161 )
6162 .map_err(RouterError::Forwarding)?;
6163 if !self.observe_module_control_completion(completion) {
6164 debug!(
6165 connection_id = connection_id.get(),
6166 corr = frame.header.corr,
6167 "dropping late malformed module-control RPC response"
6168 );
6169 }
6170 return Ok(Vec::new());
6171 }
6172 let message = format!("malformed route.bind response body: {err}");
6173 secondary_error = Some(control_error_frame(
6174 &frame,
6175 "invalid_control_body",
6176 message.clone(),
6177 )?);
6178 RouteBindRelayOutcome::ModuleGone(message)
6179 }
6180 },
6181 FrameType::Error => {
6182 if self
6183 .forwarding
6184 .pending_module_control_op(connection_id, frame.header.corr)
6185 .map_err(RouterError::Forwarding)?
6186 .is_some()
6187 {
6188 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6189 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6190 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6191 "malformed module-control ERROR body: {err}"
6192 )),
6193 };
6194 let completion = self
6195 .forwarding
6196 .complete_module_control_rpc(
6197 connection_id,
6198 frame.header.corr,
6199 None,
6200 outcome,
6201 )
6202 .map_err(RouterError::Forwarding)?;
6203 if !self.observe_module_control_completion(completion) {
6204 debug!(
6205 connection_id = connection_id.get(),
6206 corr = frame.header.corr,
6207 "dropping late or unknown module-control RPC error"
6208 );
6209 }
6210 return Ok(Vec::new());
6211 }
6212 match serde_json::from_slice::<ErrorBody>(&frame.body) {
6213 Ok(body) => RouteBindRelayOutcome::Rejected(body),
6214 Err(err) => {
6215 let message = format!("malformed route.bind ERROR body: {err}");
6216 secondary_error = Some(control_error_frame(
6217 &frame,
6218 "invalid_control_body",
6219 message.clone(),
6220 )?);
6221 RouteBindRelayOutcome::ModuleGone(message)
6222 }
6223 }
6224 }
6225 ty => {
6226 return Ok(vec![control_error_frame(
6227 &frame,
6228 "unsupported_control_frame",
6229 format!("unsupported module channel-0 frame {ty:?}"),
6230 )?])
6231 }
6232 };
6233
6234 let settled =
6235 self.forwarding
6236 .complete_pending_relay(connection_id, frame.header.corr, outcome);
6237 let completion = match settled {
6238 Ok(completion) => completion,
6239 Err(err) => {
6240 self.refuse_to_end_module_connection_for_a_client(
6241 connection_id,
6242 frame.header.corr,
6243 err,
6244 )?;
6245 return Ok(secondary_error.into_iter().collect());
6246 }
6247 };
6248 if let Some(target) = completion.abandoned.as_ref() {
6249 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6250 }
6251 if !completion.settled {
6252 debug!(
6253 connection_id = connection_id.get(),
6254 corr = frame.header.corr,
6255 frame_type = ?frame.header.ty,
6256 "dropping late or unknown route.bind relay response"
6257 );
6258 }
6259 Ok(secondary_error.into_iter().collect())
6260 }
6261
6262 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6263 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6264 self.cleanup_connection_with_end_reason(
6268 connection_id,
6269 RegistrationEndReason::ExplicitGoodbye,
6270 )
6271 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6272 Ok(Vec::new())
6273 }
6274}
6275
6276impl Default for ControlHandler {
6277 fn default() -> Self {
6278 Self::new(Arc::new(Registry::default()))
6279 }
6280}
6281
6282impl crate::supervise::SwapPromotionObserver for ControlHandler {
6283 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6284 self.apply_registration_capabilities(registration);
6285 }
6286}
6287
6288fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6289 CapabilityRequirementStatus {
6290 consumer: status.consumer,
6291 capability: status.capability,
6292 need: match status.need {
6293 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6294 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6295 },
6296 verdict: status.verdict.as_str().to_string(),
6297 episode_seq: status.episode_seq,
6298 config_satisfiable: status.config_satisfiable,
6299 runtime_available: status.runtime_available,
6300 detail: status.detail,
6301 }
6302}
6303
6304fn append_capability_problem_detail(
6305 detail: Option<String>,
6306 capability_detail: Option<String>,
6307) -> Option<String> {
6308 match (detail, capability_detail) {
6309 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6310 (Some(detail), None) => Some(detail),
6311 (None, Some(capability_detail)) => Some(capability_detail),
6312 (None, None) => None,
6313 }
6314}
6315
6316fn subc_ops() -> Vec<String> {
6317 SUBC_CONTROL_OPS
6318 .iter()
6319 .map(|op| (*op).to_string())
6320 .collect()
6321}
6322
6323fn module_subc_ops() -> Vec<String> {
6324 SUBC_CONTROL_OPS
6325 .iter()
6326 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6327 .map(|op| (*op).to_string())
6328 .collect()
6329}
6330
6331#[cfg(test)]
6332fn module_baseline_control_ops() -> Vec<String> {
6333 MODULE_BASELINE_CONTROL_OPS
6334 .iter()
6335 .map(|op| (*op).to_string())
6336 .collect()
6337}
6338
6339fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6340 let mut seen = HashSet::new();
6341 let mut effective = Vec::new();
6342 for op in MODULE_BASELINE_CONTROL_OPS {
6343 if seen.insert((*op).to_string()) {
6344 effective.push((*op).to_string());
6345 }
6346 }
6347 for op in declared.unwrap_or_default() {
6348 if seen.insert(op.clone()) {
6349 effective.push(op);
6350 }
6351 }
6352 effective
6353}
6354
6355fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6356 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6357}
6358
6359fn target_module_id(target: &RouteTarget) -> &str {
6360 match target {
6361 RouteTarget::ToolProvider { module_id }
6362 | RouteTarget::ManagementSurface { module_id }
6363 | RouteTarget::InternalService { module_id, .. } => module_id,
6364 }
6365}
6366
6367fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6368 roles.iter().any(|role| match (target, role) {
6369 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6370 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6371 (
6372 RouteTarget::InternalService { service_id, .. },
6373 ProviderRole::InternalService {
6374 service_id: provided,
6375 ..
6376 },
6377 ) => service_id == provided,
6378 _ => false,
6379 })
6380}
6381
6382fn is_routable_role(role: &ProviderRole) -> bool {
6383 matches!(
6384 role,
6385 ProviderRole::ToolProvider { .. }
6386 | ProviderRole::ManagementSurface { .. }
6387 | ProviderRole::InternalService { .. }
6388 )
6389}
6390
6391#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6392enum ControlRequestBodyError {
6393 UnknownOp,
6394 InvalidBody,
6395}
6396
6397#[derive(Debug, Deserialize)]
6398struct ControlOpProbe {
6399 op: String,
6400}
6401
6402const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6405
6406fn is_known_module_push_op(body: &[u8]) -> bool {
6407 serde_json::from_slice::<ControlOpProbe>(body)
6408 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6409 .unwrap_or(false)
6410}
6411
6412fn is_known_module_request_op(body: &[u8]) -> bool {
6413 serde_json::from_slice::<ControlOpProbe>(body)
6414 .map(|probe| is_module_to_subc_op(&probe.op))
6415 .unwrap_or(false)
6416}
6417
6418fn is_module_to_subc_op(op: &str) -> bool {
6419 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6420}
6421
6422fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6423 debug!(
6424 op = %op,
6425 connection_id = connection_id.get(),
6426 corr,
6427 "control dispatch"
6428 );
6429}
6430
6431fn log_slow_control_dispatch(
6432 dispatch_started_at: Option<StdInstant>,
6433 op: &'static str,
6434 connection_id: ConnectionId,
6435 corr: u64,
6436) {
6437 let Some(dispatch_started_at) = dispatch_started_at else {
6438 return;
6439 };
6440 let elapsed = dispatch_started_at.elapsed();
6441 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6442 warn!(
6443 op = %op,
6444 connection_id = connection_id.get(),
6445 corr,
6446 elapsed_ms = elapsed.as_millis() as u64,
6447 "slow control dispatch"
6448 );
6449 }
6450}
6451
6452fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6453 match request {
6454 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6455 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6456 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6457 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6458 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6459 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6460 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6461 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6462 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6463 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6464 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6465 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6466 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6467 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6468 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6469 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6470 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6471 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6472 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6473 }
6474}
6475
6476fn principal_label(principal: &Principal) -> String {
6477 match principal {
6478 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6479 Principal::Direct => "direct".to_string(),
6480 other => format!("{other:?}"),
6481 }
6482}
6483
6484fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6485 match request {
6486 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6487 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6488 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6489 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6490 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6491 }
6492}
6493
6494fn parse_client_control_request(
6495 body: &[u8],
6496) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6497 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6498 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6499 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6500 ControlRequestBodyError::InvalidBody
6501 }
6502 Ok(_) => ControlRequestBodyError::UnknownOp,
6503 Err(_) => ControlRequestBodyError::InvalidBody,
6504 };
6505 (err, classification)
6506 })
6507}
6508
6509fn parse_module_control_request_from_module(
6510 body: &[u8],
6511) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6512 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6513 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6514 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6515 Ok(_) => ControlRequestBodyError::UnknownOp,
6516 Err(_) => ControlRequestBodyError::InvalidBody,
6517 };
6518 (err, classification)
6519 })
6520}
6521
6522#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6523enum ProviderRoleKind {
6524 ToolProvider,
6525 PipelineStage,
6526 ManagementSurface,
6527 InternalService,
6528}
6529
6530fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6531 match role {
6532 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6533 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6534 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6535 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
6536 }
6537}
6538
6539fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
6540 roles.iter().map(provider_role_kind).collect()
6541}
6542
6543const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
6546
6547#[derive(Debug, Default, PartialEq, Eq)]
6549struct ScopeOutcomeCounts {
6550 created: usize,
6551 replaced: usize,
6552 updated: usize,
6553 unchanged: usize,
6554 refused: usize,
6555}
6556
6557impl ScopeOutcomeCounts {
6558 fn of(results: &[ScopeRecordResult]) -> Self {
6559 let mut counts = Self::default();
6560 for result in results {
6561 let slot = match result.outcome {
6562 ScopeRecordOutcome::Created => &mut counts.created,
6563 ScopeRecordOutcome::Replaced => &mut counts.replaced,
6564 ScopeRecordOutcome::Updated => &mut counts.updated,
6565 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
6566 ScopeRecordOutcome::Refused => &mut counts.refused,
6567 };
6568 *slot += 1;
6569 }
6570 counts
6571 }
6572}
6573
6574#[cfg(test)]
6575mod scope_outcome_count_tests {
6576 use super::*;
6577
6578 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
6579 ScopeRecordResult {
6580 scope_ref: "r".to_string(),
6581 scope_epoch: 1,
6582 outcome,
6583 code: None,
6584 message: None,
6585 version: None,
6586 parent_state: None,
6587 }
6588 }
6589
6590 #[test]
6593 fn every_outcome_is_counted_in_its_own_field() {
6594 let results = [
6595 result(ScopeRecordOutcome::Created),
6596 result(ScopeRecordOutcome::Created),
6597 result(ScopeRecordOutcome::Replaced),
6598 result(ScopeRecordOutcome::Updated),
6599 result(ScopeRecordOutcome::Unchanged),
6600 result(ScopeRecordOutcome::Refused),
6601 result(ScopeRecordOutcome::Refused),
6602 result(ScopeRecordOutcome::Refused),
6603 ];
6604 assert_eq!(
6605 ScopeOutcomeCounts::of(&results),
6606 ScopeOutcomeCounts {
6607 created: 2,
6608 replaced: 1,
6609 updated: 1,
6610 unchanged: 1,
6611 refused: 3,
6612 }
6613 );
6614 }
6615}
6616
6617fn capability_census_trigger(
6621 old: Option<&CapabilityDeclarations>,
6622 new: Option<&CapabilityDeclarations>,
6623) -> bool {
6624 let old_provides = old
6625 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
6626 .unwrap_or_default();
6627 let old_denies = old
6628 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
6629 .unwrap_or_default();
6630 let new = new.cloned().unwrap_or(CapabilityDeclarations {
6631 provides: Vec::new(),
6632 requires: Vec::new(),
6633 must_never_reach: Vec::new(),
6634 });
6635
6636 new.provides
6637 .iter()
6638 .any(|capability| !old_provides.contains(capability))
6639 || new
6640 .must_never_reach
6641 .iter()
6642 .any(|capability| !old_denies.contains(capability))
6643}
6644
6645fn denied_capability<'a>(
6648 opening_manifest: &'a ModuleManifest,
6649 target_manifest: &ModuleManifest,
6650) -> Option<&'a str> {
6651 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
6652 let target_capabilities = target_manifest.capabilities.as_ref()?;
6653 opening_capabilities
6654 .must_never_reach
6655 .iter()
6656 .find(|denied| {
6657 target_capabilities
6658 .provides
6659 .iter()
6660 .any(|provided| provided == *denied)
6661 })
6662 .map(String::as_str)
6663}
6664
6665fn catalog_update_frozen_field_message(
6666 registered: &ModuleManifest,
6667 provides: &[ProviderRole],
6668) -> Option<String> {
6669 let old_has_provides = !registered.provides.is_empty();
6670 let new_has_provides = !provides.is_empty();
6671 if old_has_provides != new_has_provides {
6672 return Some(format!(
6673 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
6674 registered.module_id
6675 ));
6676 }
6677
6678 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
6679 return Some(format!(
6680 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
6681 registered.module_id
6682 ));
6683 }
6684
6685 let registered_concurrency = manifest_concurrency(registered);
6686 let mut candidate = registered.clone();
6687 candidate.provides = provides.to_vec();
6688 let candidate_concurrency = manifest_concurrency(&candidate);
6689 if candidate_concurrency != registered_concurrency {
6690 return Some(format!(
6691 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
6692 registered.module_id, registered_concurrency, candidate_concurrency
6693 ));
6694 }
6695
6696 None
6699}
6700
6701fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
6702 manifest.provides.iter().any(is_routable_role)
6703}
6704
6705fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
6711 manifest
6712 .provides
6713 .iter()
6714 .find_map(|provider| match provider {
6715 ProviderRole::ToolProvider { concurrency, .. }
6716 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
6717 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
6718 })
6719 .unwrap_or(Concurrency::ModuleManaged)
6720}
6721
6722fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
6729 let has_management_surface = manifest
6730 .provides
6731 .iter()
6732 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
6733 if !has_management_surface {
6734 return false;
6735 }
6736 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
6737 return false;
6738 };
6739 let Some(provides) = raw
6740 .get("manifest")
6741 .and_then(|manifest| manifest.get("provides"))
6742 .and_then(serde_json::Value::as_array)
6743 else {
6744 return false;
6745 };
6746 provides.iter().any(|role| {
6751 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
6752 && role.get("concurrency").is_none()
6753 })
6754}
6755
6756fn negotiate_version(peer_version: u8) -> Result<u8, String> {
6757 if peer_version != PROTOCOL_VERSION {
6758 return Err(format!(
6759 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
6760 ));
6761 }
6762 Ok(PROTOCOL_VERSION)
6763}
6764
6765fn pong(frame: &Frame) -> Result<Frame, RouterError> {
6766 Frame::build_with_version(
6767 response_version(frame),
6768 FrameType::Pong,
6769 frame.header.flags,
6770 0,
6771 0,
6772 frame.header.corr,
6773 Vec::new(),
6774 )
6775 .map_err(RouterError::FrameBuild)
6776}
6777
6778pub(crate) fn control_error_frame(
6779 frame: &Frame,
6780 code: &'static str,
6781 message: impl Into<String>,
6782) -> Result<Frame, RouterError> {
6783 control_error_body_frame(
6784 frame,
6785 ErrorBody {
6786 code: code.to_string(),
6787 message: message.into(),
6788 detail: None,
6789 },
6790 )
6791}
6792
6793fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
6794 let body = serde_json::to_vec(&error).map_err(|err| {
6795 RouterError::backend(
6796 0,
6797 frame.header.corr,
6798 format!("failed to encode control ERROR: {err}"),
6799 )
6800 })?;
6801
6802 Frame::build_with_version(
6803 response_version(frame),
6804 FrameType::Error,
6805 control_flags(),
6806 0,
6807 0,
6808 frame.header.corr,
6809 body,
6810 )
6811 .map_err(RouterError::FrameBuild)
6812}
6813
6814fn control_response_body_frame<T: Serialize>(
6815 frame: &Frame,
6816 reply: &T,
6817 label: &'static str,
6818) -> Result<Frame, RouterError> {
6819 let body = serde_json::to_vec(reply).map_err(|err| {
6820 RouterError::backend(
6821 0,
6822 frame.header.corr,
6823 format!("failed to encode {label}: {err}"),
6824 )
6825 })?;
6826
6827 Frame::build_with_version(
6828 response_version(frame),
6829 FrameType::Response,
6830 control_flags(),
6831 0,
6832 0,
6833 frame.header.corr,
6834 body,
6835 )
6836 .map_err(RouterError::FrameBuild)
6837}
6838
6839fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6869 match err {
6870 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6871 ForwardingError::NoModuleConnection => "target_unavailable",
6872 ForwardingError::ModuleReloading { .. } => "module_reloading",
6873 ForwardingError::ClientRouteChannelExhausted { .. }
6874 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6875 ForwardingError::StaleModuleEndpoint
6876 | ForwardingError::UnknownReservation { .. }
6877 | ForwardingError::ConnectionClosing { .. }
6878 | ForwardingError::ClientEgressClosed { .. }
6879 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
6880 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
6883 ForwardingError::RelayCorrelationExhausted
6884 | ForwardingError::RouteOpenBuild(_)
6885 | ForwardingError::Poisoned => "forwarding_error",
6886 }
6887}
6888
6889fn response_version(frame: &Frame) -> u8 {
6890 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
6891 frame.header.ver
6892 } else {
6893 PROTOCOL_VERSION
6894 }
6895}
6896
6897fn control_flags() -> Flags {
6898 Flags::new(false, Priority::Passive, false)
6899}
6900
6901fn send_goodbye_target_best_effort(
6906 counters: &DaemonCounters,
6907 target: &GoodbyeTarget,
6908 context: &'static str,
6909) {
6910 let Ok(frame) = Frame::build_with_version(
6911 target.negotiated_ver,
6912 FrameType::Goodbye,
6913 control_flags(),
6914 target.channel,
6915 target.epoch,
6916 0,
6917 Vec::new(),
6918 ) else {
6919 return;
6920 };
6921 crate::forwarding::send_module_route_goodbye(
6922 counters,
6923 &target.sink,
6924 frame,
6925 target.module_id.as_deref(),
6926 context,
6927 );
6928}
6929
6930pub(crate) fn send_route_control_pushes(
6931 forwarding: &ForwardingTable,
6932 routes: Vec<EndpointRoute>,
6933 push: ClientControlPush,
6934) {
6935 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
6936 for route in routes {
6937 let target = route.goodbye_target;
6938 if let Some((existing, channels)) = targets
6939 .iter_mut()
6940 .find(|(existing, _)| existing.connection_id == target.connection_id)
6941 {
6942 debug_assert_eq!(
6943 existing.negotiated_ver, target.negotiated_ver,
6944 "one connection cannot negotiate multiple frame versions"
6945 );
6946 if !channels.contains(&target.channel) {
6947 channels.push(target.channel);
6948 }
6949 continue;
6950 }
6951 let channel = target.channel;
6952 targets.push((target, vec![channel]));
6953 }
6954 for (target, mut channels) in targets {
6955 channels.sort_unstable();
6956 let mut push = push.clone();
6957 match &mut push {
6958 ClientControlPush::RouteClosing {
6959 channels: covered, ..
6960 }
6961 | ClientControlPush::RouteClosed {
6962 channels: covered, ..
6963 } => *covered = channels,
6964 }
6965 let body = match serde_json::to_vec(&push) {
6966 Ok(body) => body,
6967 Err(err) => {
6968 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
6969 continue;
6970 }
6971 };
6972 let frame = match Frame::build_with_version(
6973 target.negotiated_ver,
6974 FrameType::Push,
6975 control_flags(),
6976 0,
6977 0,
6978 0,
6979 body.clone(),
6980 ) {
6981 Ok(frame) => frame,
6982 Err(err) => {
6983 warn!(
6984 route_channel = target.channel,
6985 error = %err,
6986 "failed to build route lifecycle control PUSH frame"
6987 );
6988 continue;
6989 }
6990 };
6991 if let Err(err) = target.sink.try_send(frame) {
6992 if target.close_on_delivery_failure() {
6993 warn!(
6994 target_connection_id = target.connection_id.get(),
6995 route_channel = target.channel,
6996 error = %err,
6997 "route lifecycle control PUSH was not delivered to client; closing target connection"
6998 );
6999 let _ = forwarding.escalate_client_delivery_failure(
7000 target.connection_id,
7001 target.channel,
7002 target.epoch,
7003 CloseReason::new(
7004 "route_lifecycle_push_delivery_failed",
7005 format!(
7006 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7007 target.channel
7008 ),
7009 ),
7010 crate::forwarding::UndeliveredFrame {
7011 module_id: target.module_id.as_deref(),
7012 sink: &target.sink,
7013 },
7014 );
7015 }
7016 }
7017 }
7018}
7019
7020#[cfg(test)]
7021mod tests {
7022 mod activity_read {
7023 include!("activity_read_tests.rs");
7024 }
7025 #[cfg(unix)]
7026 #[tokio::test]
7027 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7028 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7029 let path = dir.join("subc.jsonc");
7030 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7031 "program":"/bin/sleep","args":["60"],"protocol":"none",
7032 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7033 }}}).to_string()).unwrap();
7034 let mut configured = crate::daemon_config::load(&path)
7035 .unwrap()
7036 .unwrap()
7037 .modules
7038 .pop()
7039 .unwrap();
7040 let registry = std::sync::Arc::new(crate::Registry::default());
7041 let handle = crate::SupervisorHandle::new();
7042 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7043 .with_handle(handle.clone());
7044 let module = supervisor
7045 .supervise_configured_with_health(
7046 configured.module_spec(),
7047 true,
7048 configured.health.clone(),
7049 None,
7050 configured.restart,
7051 )
7052 .unwrap();
7053 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7054 let before = module.status().unwrap().pid;
7055 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7056 configured.health.cadence = std::time::Duration::from_secs(3600);
7057 let health_only = handler
7058 .reconcile_supervised_modules(&supervisor, vec![configured.clone()], false)
7059 .await
7060 .unwrap();
7061 assert!(
7062 health_only.changed_pending_reload.is_empty(),
7063 "health policy is already applied live"
7064 );
7065 assert_eq!(module.status().unwrap().pid, before);
7066 assert_eq!(
7067 module.configuration().unwrap().1.http,
7068 configured.health.http
7069 );
7070 configured.args = vec!["61".into()];
7071 let launch = handler
7072 .reconcile_supervised_modules(&supervisor, vec![configured], false)
7073 .await
7074 .unwrap();
7075 assert_eq!(launch.changed_pending_reload, ["stock"]);
7076 assert_eq!(
7077 module.status().unwrap().pid,
7078 before,
7079 "a launch edit is stored until reload"
7080 );
7081 module.drain().await.unwrap();
7082 }
7083 use cortexkit_test_support::ScratchDir;
7084 use std::{
7085 collections::BTreeMap,
7086 fmt,
7087 path::PathBuf,
7088 sync::{Arc, Mutex},
7089 time::Duration,
7090 };
7091
7092 use serde_json::{json, Value};
7093 use subc_protocol::{
7094 manifest::{
7095 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7096 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7097 },
7098 session::HealthStatus,
7099 FrameType,
7100 };
7101
7102 use super::*;
7103 use crate::{
7104 forwarding::{DataRoute, DataRouteState},
7105 registry::ChannelState,
7106 router::FrameSink,
7107 stderr_tail::DEFAULT_MAX_LINE_BYTES,
7108 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7109 RouteCtx, Router,
7110 };
7111 use tokio::{
7112 sync::mpsc,
7113 time::{sleep, Instant},
7114 };
7115 use tracing::{
7116 field::{Field, Visit},
7117 Event, Subscriber,
7118 };
7119 use tracing_subscriber::{layer::Context, prelude::*, Layer};
7120
7121 fn fake_aft_stub_path() -> PathBuf {
7140 let mut path = std::env::current_exe().expect("current_exe available in tests");
7141 path.pop(); path.pop(); path.push(if cfg!(windows) {
7144 "fake-aft-stub.exe"
7145 } else {
7146 "fake-aft-stub"
7147 });
7148 assert!(
7149 path.exists(),
7150 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7151 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7152 path.display()
7153 );
7154 path
7155 }
7156
7157 fn client_retries(code: &str) -> bool {
7162 subc_protocol::error_codes::is_retryable_route_open(code)
7163 }
7164
7165 #[test]
7176 fn retryability_of_forwarding_codes_matches_the_failure() {
7177 let transient = [
7180 ForwardingError::NoModuleConnection,
7181 ForwardingError::ModuleReloading {
7182 module_id: "m".into(),
7183 },
7184 ForwardingError::StaleModuleEndpoint,
7185 ForwardingError::UnknownReservation {
7186 client_channel: 1,
7187 module_channel: 1,
7188 },
7189 ForwardingError::ConnectionClosing {
7190 connection_id: ConnectionId::new(1),
7191 },
7192 ForwardingError::ClientEgressClosed {
7193 connection_id: ConnectionId::new(1),
7194 },
7195 ForwardingError::ModuleEgressUnavailable {
7196 connection_id: ConnectionId::new(1),
7197 },
7198 ];
7199 for err in transient {
7200 let code = forwarding_error_code(&err);
7201 assert!(
7202 client_retries(code),
7203 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7204 );
7205 }
7206
7207 let permanent = [
7212 ForwardingError::ConnectionRoleConflict {
7213 connection_id: ConnectionId::new(1),
7214 },
7215 ForwardingError::ClientRouteChannelExhausted {
7216 connection_id: ConnectionId::new(1),
7217 },
7218 ForwardingError::ModuleRouteChannelExhausted {
7219 endpoint: ModuleEndpointId {
7220 connection_id: ConnectionId::new(1),
7221 generation: 1,
7222 },
7223 },
7224 ForwardingError::RelayCorrelationExhausted,
7225 ForwardingError::RouteOpenBuild("x".into()),
7226 ForwardingError::Poisoned,
7227 ];
7228 for err in permanent {
7229 let code = forwarding_error_code(&err);
7230 assert!(
7231 !client_retries(code),
7232 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7233 );
7234 }
7235 }
7236
7237 #[tokio::test]
7265 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7266 let handler = ControlHandler::default();
7267 let frame =
7268 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7269
7270 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7274 assert_eq!(
7275 stamped,
7276 Principal::Direct,
7277 "a caller that proved nothing must not be stamped as a supervised module"
7278 );
7279
7280 let forged = handler
7285 .route_open_principal(
7286 &frame,
7287 Some(ConsumerIdentity {
7288 module_id: "aft".to_string(),
7289 launch_nonce: "not-a-real-nonce".to_string(),
7290 }),
7291 )
7292 .unwrap();
7293 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7294 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7295 }
7296
7297 #[test]
7314 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7315 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7316 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7317 let ClientControlRequest::RouteOpen {
7318 consumer_identity, ..
7319 } = parsed
7320 else {
7321 panic!("route.open body must parse as RouteOpen");
7322 };
7323 assert_eq!(
7324 consumer_identity,
7325 Some(ConsumerIdentity {
7326 module_id: "aft".to_string(),
7327 launch_nonce: "n".to_string(),
7328 }),
7329 "the wire field name must reach the value route_open_principal reads"
7330 );
7331 }
7332
7333 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7334 ModuleManifest::builder(module_id, "0.1.0")
7335 .protocol_ver(protocol_ver)
7336 .provides(vec![ProviderRole::ToolProvider {
7337 tools: vec![Tool {
7338 name: "read".to_string(),
7339 description: None,
7340 execution_mode: ExecutionMode::Pure,
7341 schema: json!({"type": "object"}),
7342 }],
7343 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7344 concurrency: Concurrency::ModuleManaged,
7345 emits_push: true,
7346 sub_supervises: true,
7347 }])
7348 .build()
7349 }
7350
7351 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7352 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7353 }
7354
7355 fn hello_frame_with_control_ops(
7356 module_id: &str,
7357 protocol_ver: u8,
7358 corr: u64,
7359 control_ops: Option<Vec<String>>,
7360 ) -> Frame {
7361 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7362 }
7363
7364 fn hello_frame_with_nonce(
7365 module_id: &str,
7366 protocol_ver: u8,
7367 corr: u64,
7368 launch_nonce: Option<&str>,
7369 ) -> Frame {
7370 hello_frame_full(
7371 module_id,
7372 protocol_ver,
7373 corr,
7374 None,
7375 launch_nonce.map(ToOwned::to_owned),
7376 )
7377 }
7378
7379 fn hello_frame_full(
7380 module_id: &str,
7381 protocol_ver: u8,
7382 corr: u64,
7383 control_ops: Option<Vec<String>>,
7384 launch_nonce: Option<String>,
7385 ) -> Frame {
7386 let body = serde_json::to_vec(&ModuleHelloBody {
7387 manifest: manifest(module_id, protocol_ver),
7388 protocol_ver,
7389 control_ops,
7390 launch_nonce,
7391 })
7392 .unwrap();
7393 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7394 }
7395
7396 fn non_routable_hello_frame_with_control_ops(
7397 module_id: &str,
7398 corr: u64,
7399 control_ops: Option<Vec<String>>,
7400 ) -> Frame {
7401 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
7402 manifest.provides.clear();
7403 let body = serde_json::to_vec(&ModuleHelloBody {
7404 manifest,
7405 protocol_ver: PROTOCOL_VERSION,
7406 control_ops,
7407 launch_nonce: None,
7408 })
7409 .unwrap();
7410 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7411 }
7412
7413 fn capability_grammar_hello_frame(
7414 capabilities: Value,
7415 runtime_computed: Option<Value>,
7416 corr: u64,
7417 ) -> Frame {
7418 let mut body = serde_json::to_value(ModuleHelloBody {
7419 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
7420 protocol_ver: PROTOCOL_VERSION,
7421 control_ops: None,
7422 launch_nonce: None,
7423 })
7424 .expect("HELLO body serializes");
7425 body["manifest"]["capabilities"] = capabilities;
7426 if let Some(runtime_computed) = runtime_computed {
7427 body["runtime_computed"] = runtime_computed;
7428 }
7429 Frame::build(
7430 FrameType::Hello,
7431 control_flags(),
7432 0,
7433 0,
7434 corr,
7435 serde_json::to_vec(&body).expect("HELLO body reserializes"),
7436 )
7437 .expect("HELLO frame builds")
7438 }
7439
7440 fn channel_request(channel: u16, corr: u64) -> Frame {
7441 Frame::build(
7442 FrameType::Request,
7443 Flags::new(true, Priority::Interactive, false),
7444 channel,
7445 0,
7446 corr,
7447 b"opaque".to_vec(),
7448 )
7449 .unwrap()
7450 }
7451
7452 fn route_ctx(
7453 connection_id: ConnectionId,
7454 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7455 let (tx, rx) = mpsc::channel(8);
7456 (
7457 RouteCtx {
7458 connection_id,
7459 egress: FrameSink::new(tx),
7460 },
7461 rx,
7462 )
7463 }
7464
7465 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7466 serde_json::from_slice(&frame.body).unwrap()
7467 }
7468
7469 async fn hello_via_sink(
7474 handler: &ControlHandler,
7475 ctx: &RouteCtx,
7476 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7477 hello: Frame,
7478 ) -> Frame {
7479 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7480 assert!(
7481 replies.is_empty(),
7482 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7483 );
7484 let ack = rx
7485 .try_recv()
7486 .expect("HELLO_ACK is queued on the module sink")
7487 .frame;
7488 assert_eq!(ack.header.ty, FrameType::HelloAck);
7489 ack
7490 }
7491
7492 fn parse_error(frame: &Frame) -> Value {
7493 serde_json::from_slice(&frame.body).unwrap()
7494 }
7495
7496 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7497 serde_json::from_slice(&frame.body).unwrap()
7498 }
7499
7500 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7501 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7502 route_channel,
7503 route_epoch: 0,
7504 kind,
7505 })
7506 .unwrap();
7507 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7508 }
7509
7510 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7511 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7512 module_id: module_id.to_string(),
7513 })
7514 .unwrap();
7515 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7516 }
7517
7518 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7519 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7520 }
7521
7522 fn route_open_frame_with_consumer_capabilities(
7523 corr: u64,
7524 module_id: &str,
7525 project_root: ScratchDir,
7526 consumer_capabilities: Option<Vec<String>>,
7527 ) -> Frame {
7528 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7529 target: RouteTarget::ToolProvider {
7530 module_id: module_id.to_string(),
7531 },
7532 identity: BindIdentity::new(
7533 project_root.path().to_path_buf(),
7534 "unit".to_string(),
7535 "session".to_string(),
7536 ),
7537 consumer_identity: None,
7538 consumer_capabilities,
7539 role_versions: None,
7540 admission_facts: None,
7541 scope: None,
7542 })
7543 .unwrap();
7544 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7545 }
7546
7547 fn route_open_frame_with_role_versions(
7548 corr: u64,
7549 module_id: &str,
7550 project_root: ScratchDir,
7551 role_versions: Option<BTreeMap<String, String>>,
7552 ) -> Frame {
7553 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7554 target: RouteTarget::ToolProvider {
7555 module_id: module_id.to_string(),
7556 },
7557 identity: BindIdentity::new(
7558 project_root.path().to_path_buf(),
7559 "unit".to_string(),
7560 format!("session-{corr}"),
7561 ),
7562 consumer_identity: None,
7563 consumer_capabilities: None,
7564 role_versions,
7565 admission_facts: None,
7566 scope: None,
7567 })
7568 .unwrap();
7569 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7570 }
7571
7572 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
7573 entries
7574 .iter()
7575 .map(|(role, version)| (role.to_string(), version.to_string()))
7576 .collect()
7577 }
7578
7579 fn route_open_frame_with_admission_facts(
7580 corr: u64,
7581 module_id: &str,
7582 project_root: ScratchDir,
7583 consumer_identity: Option<subc_control::ConsumerIdentity>,
7584 facts: Option<Value>,
7585 ) -> Frame {
7586 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7587 target: RouteTarget::ToolProvider {
7588 module_id: module_id.to_string(),
7589 },
7590 identity: BindIdentity::new(
7591 project_root.path().to_path_buf(),
7592 "unit".to_string(),
7593 format!("session-{corr}"),
7594 ),
7595 consumer_identity,
7596 consumer_capabilities: None,
7597 role_versions: None,
7598 admission_facts: facts,
7599 scope: None,
7600 })
7601 .unwrap();
7602 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7603 }
7604
7605 #[derive(Clone, Default)]
7606 struct EventCapture {
7607 events: Arc<Mutex<Vec<CapturedEvent>>>,
7608 }
7609
7610 #[derive(Clone, Debug)]
7611 struct CapturedEvent {
7612 target: String,
7613 level: tracing::Level,
7614 fields: BTreeMap<String, String>,
7615 }
7616
7617 impl EventCapture {
7618 fn events(&self) -> Vec<CapturedEvent> {
7619 self.events.lock().unwrap().clone()
7620 }
7621 }
7622
7623 impl<S> Layer<S> for EventCapture
7624 where
7625 S: Subscriber,
7626 {
7627 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
7628 let mut visitor = EventFieldVisitor::default();
7629 event.record(&mut visitor);
7630 self.events.lock().unwrap().push(CapturedEvent {
7631 target: event.metadata().target().to_string(),
7632 level: *event.metadata().level(),
7633 fields: visitor.fields,
7634 });
7635 }
7636 }
7637
7638 #[derive(Default)]
7639 struct EventFieldVisitor {
7640 fields: BTreeMap<String, String>,
7641 }
7642
7643 impl Visit for EventFieldVisitor {
7644 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
7645 self.fields
7646 .insert(field.name().to_string(), format!("{value:?}"));
7647 }
7648 }
7649
7650 fn health_response(corr: u64, status: HealthStatus) -> Frame {
7651 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
7652 status,
7653 detail: Some("warming".to_string()),
7654 metrics: Some(json!({"queue_depth": 3})),
7655 })
7656 .unwrap();
7657 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7658 }
7659
7660 fn route_bind_ack(corr: u64) -> Frame {
7661 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
7662 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7663 }
7664
7665 fn unique_project_root(label: &str) -> ScratchDir {
7666 ScratchDir::new(label)
7667 }
7668
7669 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
7670 match parse_route_poll(frame) {
7671 ClientControlResponse::RoutePoll {
7672 status: None,
7673 live: Some(live),
7674 ..
7675 } => assert_eq!(live, expected_live),
7676 other => panic!("unexpected route.poll response: {other:?}"),
7677 }
7678 }
7679
7680 fn bind_liveness_route(
7681 registry: &Registry,
7682 forwarding: &ForwardingTable,
7683 module_id: &str,
7684 ) -> (RouteCtx, u16, u32) {
7685 let module_connection = ConnectionId::new(101);
7686 let client_connection = ConnectionId::new(202);
7687 let registration = registry
7688 .register_with_control_ops(
7689 manifest(module_id, PROTOCOL_VERSION),
7690 PROTOCOL_VERSION,
7691 module_connection,
7692 module_baseline_control_ops(),
7693 )
7694 .unwrap();
7695 let (module_tx, _module_rx) = mpsc::channel(8);
7696 let endpoint = forwarding
7697 .register_module_connection(
7698 module_connection,
7699 module_id.to_string(),
7700 PROTOCOL_VERSION,
7701 manifest_concurrency(®istration.manifest),
7702 FrameSink::new(module_tx),
7703 )
7704 .unwrap();
7705 let (client_ctx, _client_rx) = route_ctx(client_connection);
7706 let pending = forwarding
7707 .begin_route_bind_relay_for_test(
7708 client_connection,
7709 client_ctx.egress.clone(),
7710 1,
7711 module_id,
7712 )
7713 .unwrap();
7714 assert_eq!(pending.endpoint, endpoint);
7715 let route_channel = pending.client_channel;
7716 let route_epoch = pending.client_epoch;
7717 forwarding
7718 .complete_pending_relay(
7719 module_connection,
7720 pending.corr,
7721 RouteBindRelayOutcome::Accepted,
7722 )
7723 .unwrap();
7724 (client_ctx, route_channel, route_epoch)
7725 }
7726
7727 struct FakeProcessLiveness {
7728 live: Option<bool>,
7729 }
7730
7731 impl ModuleProcessLiveness for FakeProcessLiveness {
7732 fn process_live(&self, _module_id: &str) -> Option<bool> {
7733 self.live
7734 }
7735 }
7736
7737 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7738 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
7739 {
7740 let registry = Arc::new(Registry::default());
7741 let supervisor_handle = SupervisorHandle::new();
7742 let supervisor = Supervisor::new_for_test(
7743 Arc::clone(®istry),
7744 RestartPolicy::new(1, Duration::from_millis(10)),
7745 )
7746 .with_handle(supervisor_handle.clone());
7747 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
7748 let module = supervisor
7749 .spawn(ModuleSpec {
7750 module_id: "stderr-tail-wire".to_string(),
7751 program: fake_aft_stub_path(),
7752 args: Vec::new(),
7753 env: vec![
7754 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
7755 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
7756 ],
7757 reserved: false,
7758 reserved_prefixes: Vec::new(),
7759 protocol: ModuleProtocol::Subc,
7760 overlap: Default::default(),
7761 })
7762 .unwrap();
7763
7764 let deadline = Instant::now() + Duration::from_secs(5);
7765 loop {
7766 let tail = module.stderr_tail(None, None);
7767 if tail
7768 .entries
7769 .iter()
7770 .any(|entry| matches!(entry, TailEntry::ProcessStart))
7771 && tail.entries.iter().any(|entry| {
7772 matches!(
7773 entry,
7774 TailEntry::Line {
7775 truncated: true,
7776 ..
7777 }
7778 )
7779 })
7780 {
7781 break;
7782 }
7783 assert!(
7784 Instant::now() < deadline,
7785 "module did not produce a truncated line and restart boundary: {tail:?}"
7786 );
7787 sleep(Duration::from_millis(10)).await;
7788 }
7789
7790 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7791 let request = ClientControlRequest::SupervisorStderrTail {
7792 module_id: "stderr-tail-wire".to_string(),
7793 max_lines: None,
7794 max_bytes: None,
7795 };
7796 let frame = Frame::build(
7797 FrameType::Request,
7798 control_flags(),
7799 0,
7800 0,
7801 1,
7802 serde_json::to_vec(&request).unwrap(),
7803 )
7804 .unwrap();
7805 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7806 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7807 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
7808 serde_json::from_slice(&responses[0].body).unwrap()
7809 else {
7810 panic!("expected supervisor.stderr_tail response");
7811 };
7812
7813 assert!(
7814 tail.entries
7815 .iter()
7816 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
7817 "the control response lost the restart boundary"
7818 );
7819 let Some(StderrTailEntry::Line {
7820 text,
7821 truncated,
7822 at_ms,
7823 }) = tail.entries.iter().find(|entry| {
7824 matches!(
7825 entry,
7826 StderrTailEntry::Line {
7827 truncated: true,
7828 ..
7829 }
7830 )
7831 })
7832 else {
7833 panic!("the control response lost the truncated line");
7834 };
7835 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
7836 assert!(*truncated);
7837 assert!(
7838 at_ms.is_some(),
7839 "the control response lost the line's capture time"
7840 );
7841 }
7842
7843 #[tokio::test(flavor = "current_thread")]
7848 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
7849 let dir = ScratchDir::new("terminals-off-worker");
7850 let journal_path = dir.join("terminals.jsonl");
7851 let registry = Arc::new(Registry::default());
7852 let supervisor_handle = SupervisorHandle::new();
7853 let supervisor =
7854 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7855 .with_handle(supervisor_handle.clone())
7856 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7857 let module = supervisor
7858 .spawn(ModuleSpec {
7859 module_id: "terminal-off-worker".to_string(),
7860 program: fake_aft_stub_path(),
7861 args: Vec::new(),
7862 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7863 reserved: false,
7864 reserved_prefixes: Vec::new(),
7865 protocol: ModuleProtocol::Subc,
7866 overlap: Default::default(),
7867 })
7868 .unwrap();
7869 let deadline = Instant::now() + Duration::from_secs(5);
7870 while module.terminal_history().entries.len() != 2 {
7871 assert!(Instant::now() < deadline, "module did not record two exits");
7872 sleep(Duration::from_millis(10)).await;
7873 }
7874
7875 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
7876 let handler =
7877 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
7878 let frame = Frame::build(
7879 FrameType::Request,
7880 control_flags(),
7881 0,
7882 0,
7883 1,
7884 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
7885 module_id: "terminal-off-worker".to_string(),
7886 })
7887 .unwrap(),
7888 )
7889 .unwrap();
7890 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7891 let spawned_at = std::time::Instant::now();
7892 let read = tokio::spawn({
7893 let handler = Arc::clone(&handler);
7894 async move { handler.handle_control_frame(&ctx, frame).await }
7895 });
7896 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
7899 .await
7900 .unwrap()
7901 .expect("the history read reached its pause");
7902 let elapsed = spawned_at.elapsed();
7903 assert!(
7904 elapsed < Duration::from_secs(2) && !read.is_finished(),
7905 "this task could not run while the history read was paused \
7906 (resumed after {elapsed:?}, read finished: {})",
7907 read.is_finished()
7908 );
7909
7910 drop(release);
7911 let responses = read.await.unwrap().unwrap();
7912 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7913 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
7914 panic!("expected supervisor.terminals response");
7915 };
7916 assert_eq!(terminals.entries.len(), 2);
7917 assert_eq!(terminals.journal_skipped_lines, 0);
7918 assert_eq!(terminals.journal_read_errors, 0);
7919 }
7920
7921 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7922 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
7923 let registry = Arc::new(Registry::default());
7924 let supervisor_handle = SupervisorHandle::new();
7925 let supervisor =
7926 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7927 .with_handle(supervisor_handle.clone());
7928 let module = supervisor
7929 .spawn(ModuleSpec {
7930 module_id: "terminal-golden".to_string(),
7931 program: fake_aft_stub_path(),
7932 args: Vec::new(),
7933 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7934 reserved: false,
7935 reserved_prefixes: Vec::new(),
7936 protocol: ModuleProtocol::Subc,
7937 overlap: Default::default(),
7938 })
7939 .unwrap();
7940
7941 let deadline = Instant::now() + Duration::from_secs(5);
7942 while module.terminal_history().entries.len() != 2 {
7943 assert!(
7944 Instant::now() < deadline,
7945 "module did not retain two terminal exits: {:?}",
7946 module.terminal_history()
7947 );
7948 sleep(Duration::from_millis(10)).await;
7949 }
7950
7951 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7952 let request = ClientControlRequest::SupervisorTerminals {
7953 module_id: "terminal-golden".to_string(),
7954 };
7955 let frame = Frame::build(
7956 FrameType::Request,
7957 control_flags(),
7958 0,
7959 0,
7960 1,
7961 serde_json::to_vec(&request).unwrap(),
7962 )
7963 .unwrap();
7964 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7965 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7966 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
7967 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
7968 panic!("expected supervisor.terminals response");
7969 };
7970 assert_eq!(terminals.entries.len(), 2);
7971 assert_eq!(terminals.dropped, 0);
7972
7973 let mut rendered = serde_json::to_value(response).unwrap();
7974 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
7977 for (index, entry) in rendered["entries"]
7978 .as_array_mut()
7979 .expect("terminal response entries array")
7980 .iter_mut()
7981 .enumerate()
7982 {
7983 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
7984 }
7985
7986 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
7987 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
7988 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
7989 if std::env::var_os("UPDATE_GOLDEN").is_some() {
7990 std::fs::write(&golden_path, &serialized).unwrap();
7991 }
7992 let expected: Value =
7993 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
7994 assert_eq!(rendered, expected);
7995 }
7996
7997 #[test]
7998 fn hello_registers_manifest_and_returns_ack() {
7999 let registry = Arc::new(Registry::default());
8000 let handler = ControlHandler::new(Arc::clone(®istry));
8001 let conn = ConnectionId::new(1);
8002
8003 let responses = handler
8004 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8005 .unwrap();
8006
8007 assert_eq!(responses.len(), 1);
8008 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8009 assert_eq!(responses[0].header.channel, 0);
8010 assert_eq!(responses[0].header.corr, 7);
8011 let ack = parse_ack(&responses[0]);
8012 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8013 assert!(ack
8014 .subc_capabilities
8015 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8016 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8017 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8018 assert!(ack
8019 .subc_ops
8020 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8021 assert!(ack
8022 .subc_ops
8023 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8024
8025 let registration = registry.get_module("aft").unwrap().unwrap();
8026 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8027 assert_eq!(registration.state, ChannelState::Active);
8028 assert_eq!(registration.connection_id, conn);
8029 assert_eq!(registration.control_ops, module_baseline_control_ops());
8030 }
8031
8032 #[test]
8033 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8034 let invalid_identifiers = [
8035 ("case_change", "credentials-Provider/v1"),
8036 ("leading_zero", "credentials-provider/v01"),
8037 ("trailing_hyphen", "credentials-provider-/v1"),
8038 ("consecutive_hyphens", "credentials--provider/v1"),
8039 ("uppercase", "Credentials-provider/v1"),
8040 ("missing_v", "credentials-provider/1"),
8041 ("whitespace", "credentials provider/v1"),
8042 ("zero_version", "credentials-provider/v0"),
8043 ("out_of_range_version", "credentials-provider/v4294967296"),
8044 (
8045 "overlength_name",
8046 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8047 ),
8048 ];
8049 let mut cases = invalid_identifiers
8050 .into_iter()
8051 .map(|(name, identifier)| {
8052 (
8053 format!("identifier_{name}"),
8054 "capabilities.provides[0]".to_string(),
8055 identifier.to_string(),
8056 json!({ "provides": [identifier] }),
8057 None,
8058 )
8059 })
8060 .collect::<Vec<_>>();
8061 cases.extend([
8062 (
8063 "unknown_need".to_string(),
8064 "capabilities.requires[0].need".to_string(),
8065 "deferred".to_string(),
8066 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8067 None,
8068 ),
8069 (
8070 "duplicate_provides".to_string(),
8071 "capabilities.provides[1]".to_string(),
8072 "credentials-provider/v1".to_string(),
8073 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8074 None,
8075 ),
8076 (
8077 "duplicate_must_never_reach".to_string(),
8078 "capabilities.must_never_reach[1]".to_string(),
8079 "credentials-provider/v1".to_string(),
8080 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8081 None,
8082 ),
8083 (
8084 "duplicate_requires_same_need".to_string(),
8085 "capabilities.requires[1]".to_string(),
8086 "credentials-provider/v1".to_string(),
8087 json!({ "requires": [
8088 { "capability": "credentials-provider/v1", "need": "required" },
8089 { "capability": "credentials-provider/v1", "need": "required" }
8090 ] }),
8091 None,
8092 ),
8093 (
8094 "duplicate_requires_conflicting_need".to_string(),
8095 "capabilities.requires[1]".to_string(),
8096 "credentials-provider/v1".to_string(),
8097 json!({ "requires": [
8098 { "capability": "credentials-provider/v1", "need": "required" },
8099 { "capability": "credentials-provider/v1", "need": "optional" }
8100 ] }),
8101 None,
8102 ),
8103 (
8104 "capabilities_root_pointer".to_string(),
8105 "runtime_computed[0]".to_string(),
8106 "/capabilities".to_string(),
8107 json!({}),
8108 Some(json!(["/capabilities"])),
8109 ),
8110 (
8111 "capabilities_descendant_pointer".to_string(),
8112 "runtime_computed[0]".to_string(),
8113 "/capabilities/provides".to_string(),
8114 json!({}),
8115 Some(json!(["/capabilities/provides"])),
8116 ),
8117 (
8118 "malformed_pointer_without_leading_slash".to_string(),
8119 "runtime_computed[0]".to_string(),
8120 "capabilities".to_string(),
8121 json!({}),
8122 Some(json!(["capabilities"])),
8123 ),
8124 (
8125 "malformed_pointer_escape".to_string(),
8126 "runtime_computed[0]".to_string(),
8127 "/roles/~2/tools".to_string(),
8128 json!({}),
8129 Some(json!(["/roles/~2/tools"])),
8130 ),
8131 (
8132 "unknown_capabilities_field".to_string(),
8133 "capabilities.future".to_string(),
8134 "<array>".to_string(),
8135 json!({ "future": [] }),
8136 None,
8137 ),
8138 ]);
8139
8140 for (index, (name, field, value, capabilities, runtime_computed)) in
8141 cases.into_iter().enumerate()
8142 {
8143 let registry = Arc::new(Registry::default());
8144 let handler = ControlHandler::new(Arc::clone(®istry));
8145 let response = handler
8146 .handle_control(
8147 ConnectionId::new((index + 1) as u64),
8148 capability_grammar_hello_frame(
8149 capabilities,
8150 runtime_computed,
8151 index as u64 + 1,
8152 ),
8153 )
8154 .expect("invalid HELLO returns a refusal");
8155
8156 assert_eq!(response.len(), 1, "{name} must emit one refusal");
8157 let error = parse_error(&response[0]);
8158 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8159 let message = error["message"]
8160 .as_str()
8161 .expect("error message is a string");
8162 assert!(
8163 message.contains(&field),
8164 "{name}: field missing from {message}"
8165 );
8166 assert!(
8167 message.contains(&value),
8168 "{name}: value missing from {message}"
8169 );
8170 assert_eq!(
8171 registry
8172 .active_registration_count()
8173 .expect("registry reads"),
8174 0,
8175 "{name}: refused HELLO must not create a catalog entry"
8176 );
8177 }
8178 }
8179
8180 #[test]
8181 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8182 let registry = Arc::new(Registry::default());
8183 let handler = ControlHandler::new(Arc::clone(®istry));
8184 let capabilities = json!({
8185 "provides": ["credentials-provider/v1"],
8186 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8187 "must_never_reach": ["federation-transport/v1"]
8188 });
8189 let response = handler
8190 .handle_control(
8191 ConnectionId::new(99),
8192 capability_grammar_hello_frame(
8193 capabilities.clone(),
8194 Some(json!(["/roles/0/tools"])),
8195 99,
8196 ),
8197 )
8198 .expect("valid HELLO registers");
8199 assert_eq!(response[0].header.ty, FrameType::HelloAck);
8200
8201 let request = Frame::build(
8202 FrameType::Request,
8203 control_flags(),
8204 0,
8205 0,
8206 100,
8207 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8208 .expect("catalog request serializes"),
8209 )
8210 .expect("catalog request frame builds");
8211 let response = handler
8212 .handle_catalog_list(request, None)
8213 .expect("catalog list succeeds");
8214 let ClientControlResponse::CatalogList { modules, .. } =
8215 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8216 else {
8217 panic!("catalog request must return catalog.list");
8218 };
8219 assert_eq!(modules.len(), 1);
8220 assert_eq!(
8221 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8222 capabilities
8223 );
8224 }
8225
8226 #[test]
8227 fn catalog_list_mirrors_management_operation_description() {
8228 let registry = Arc::new(Registry::default());
8229 let handler = ControlHandler::new(Arc::clone(®istry));
8230 let description = "List managed records and return their identifiers and metadata.";
8231 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8232 manifest.provides = vec![ProviderRole::ManagementSurface {
8233 operations: vec![ManagementOperation {
8234 name: "records.list".to_string(),
8235 kind: ManagementOperationKind::Query,
8236 description: Some(description.to_string()),
8237 }],
8238 config_schema: json!({"type": "object"}),
8239 observability: vec![ObservabilitySurface {
8240 name: "records.stats".to_string(),
8241 kind: ObservabilityKind::Snapshot,
8242 }],
8243 identity_scope: vec![IdentityScope::Project],
8244 concurrency: Concurrency::ModuleManaged,
8245 }];
8246 registry
8247 .register_with_control_ops(
8248 manifest,
8249 PROTOCOL_VERSION,
8250 ConnectionId::new(99),
8251 Vec::new(),
8252 )
8253 .expect("described management manifest registers");
8254
8255 let request = Frame::build(
8256 FrameType::Request,
8257 control_flags(),
8258 0,
8259 0,
8260 100,
8261 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8262 .expect("catalog request serializes"),
8263 )
8264 .expect("catalog request frame builds");
8265 let response = handler
8266 .handle_catalog_list(request, None)
8267 .expect("catalog list succeeds");
8268 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8269 assert_eq!(
8270 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8271 "catalog.list must preserve the declared operation description verbatim"
8272 );
8273 }
8274
8275 #[test]
8276 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8277 let registry = Arc::new(Registry::default());
8278 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8279 [("vault".to_string(), true), ("squatter".to_string(), true)],
8280 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8281 );
8282 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8283 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8284 provides: vec!["credentials-provider/v1".to_string()],
8285 requires: Vec::new(),
8286 must_never_reach: Vec::new(),
8287 });
8288 let frame = Frame::build(
8289 FrameType::Hello,
8290 control_flags(),
8291 0,
8292 0,
8293 77,
8294 serde_json::to_vec(&ModuleHelloBody {
8295 manifest: squatter,
8296 protocol_ver: PROTOCOL_VERSION,
8297 control_ops: None,
8298 launch_nonce: None,
8299 })
8300 .expect("HELLO serializes"),
8301 )
8302 .expect("HELLO frame builds");
8303 let response = handler
8304 .handle_control(ConnectionId::new(77), frame)
8305 .expect("reserved claim receives a typed refusal");
8306 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8307 assert_eq!(
8308 registry
8309 .active_registration_count()
8310 .expect("registry reads"),
8311 0,
8312 "a reserved capability refusal must not leave a catalog entry"
8313 );
8314 }
8315
8316 #[test]
8317 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8318 for settlement in ["timeout", "inconclusive", "drop"] {
8319 let breakers = RouteBindBreakers::default();
8320 let RouteBindAdmission::Admitted {
8321 guard: mut old,
8322 probe: false,
8323 } = breakers.admit("prov")
8324 else {
8325 panic!("ordinary relay admitted")
8326 };
8327 let RouteBindAdmission::Admitted {
8328 guard: mut opener, ..
8329 } = breakers.admit("prov")
8330 else {
8331 panic!("second relay admitted")
8332 };
8333 assert!(
8334 !opener
8335 .record_timeout(1, Duration::ZERO)
8336 .unwrap()
8337 .reopened_after_probe
8338 );
8339 let RouteBindAdmission::Admitted {
8340 guard: mut probe,
8341 probe: true,
8342 } = breakers.admit("prov")
8343 else {
8344 panic!("one cooldown probe admitted")
8345 };
8346 match settlement {
8347 "timeout" => assert!(
8348 !old.record_timeout(1, Duration::ZERO)
8349 .unwrap()
8350 .reopened_after_probe
8351 ),
8352 "inconclusive" => old.record_inconclusive(),
8353 "drop" => drop(old),
8354 _ => unreachable!(),
8355 }
8356 assert!(
8357 matches!(
8358 breakers.admit("prov"),
8359 RouteBindAdmission::Refused {
8360 probe_in_flight: true,
8361 ..
8362 }
8363 ),
8364 "{settlement} of a pre-open relay cannot release the real probe"
8365 );
8366 assert!(
8367 probe
8368 .record_timeout(1, Duration::ZERO)
8369 .unwrap()
8370 .reopened_after_probe
8371 );
8372 assert!(matches!(
8373 breakers.admit("prov"),
8374 RouteBindAdmission::Admitted { probe: true, .. }
8375 ));
8376 }
8377 let breakers = RouteBindBreakers::default();
8378 let admit = || match breakers.admit("prov") {
8379 RouteBindAdmission::Admitted { guard, .. } => guard,
8380 _ => panic!("relay admitted"),
8381 };
8382 admit().record_timeout(1, Duration::ZERO);
8383 let mut old_probe = admit();
8384 breakers.reset_for_new_module_connection("prov");
8385 admit().record_timeout(1, Duration::ZERO);
8386 let _new_probe = admit();
8387 old_probe.record_inconclusive();
8388 assert!(matches!(
8389 breakers.admit("prov"),
8390 RouteBindAdmission::Refused {
8391 probe_in_flight: true,
8392 ..
8393 }
8394 ));
8395 }
8396
8397 #[tokio::test]
8398 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
8399 for candidate in [false, true] {
8400 let registry = Arc::new(Registry::default());
8401 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8402 [("vault".to_string(), true), ("squatter".to_string(), true)],
8403 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8404 );
8405 let conn = ConnectionId::new(77);
8406 let (ctx, mut rx) = route_ctx(conn);
8407 let initial = capability_manifest("squatter", &[], &[]);
8408 if candidate {
8409 registry
8410 .register_candidate_with_control_ops(
8411 initial.clone(),
8412 PROTOCOL_VERSION,
8413 conn,
8414 module_baseline_control_ops(),
8415 )
8416 .unwrap();
8417 handler
8418 .forwarding
8419 .register_candidate_module_connection(
8420 conn,
8421 "squatter".to_string(),
8422 PROTOCOL_VERSION,
8423 manifest_concurrency(&initial),
8424 ctx.egress.clone(),
8425 )
8426 .unwrap();
8427 } else {
8428 hello_via_sink(
8429 &handler,
8430 &ctx,
8431 &mut rx,
8432 hello_frame_with_manifest(initial.clone(), 1),
8433 )
8434 .await;
8435 }
8436 let response = handler
8437 .handle_control_frame(
8438 &ctx,
8439 catalog_update_with_capabilities_frame(
8440 2,
8441 capability_manifest("squatter", &["credentials-provider/v1"], &[])
8442 .capabilities
8443 .unwrap(),
8444 ),
8445 )
8446 .await
8447 .unwrap();
8448 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8449 assert_eq!(
8450 registry
8451 .get_module_by_connection(conn)
8452 .unwrap()
8453 .unwrap()
8454 .manifest,
8455 initial
8456 );
8457 }
8458 }
8459
8460 #[test]
8461 fn server_describe_surfaces_required_capability_verdict_fields() {
8462 let registry = Arc::new(Registry::default());
8463 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8464 [
8465 ("consumer".to_string(), true),
8466 ("provider".to_string(), false),
8467 ],
8468 BTreeMap::new(),
8469 );
8470 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
8471 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8472 provides: Vec::new(),
8473 requires: vec![subc_protocol::manifest::CapabilityRequirement {
8474 capability: "credentials-provider/v1".to_string(),
8475 need: subc_protocol::manifest::CapabilityNeed::Required,
8476 }],
8477 must_never_reach: Vec::new(),
8478 });
8479 let hello = Frame::build(
8480 FrameType::Hello,
8481 control_flags(),
8482 0,
8483 0,
8484 78,
8485 serde_json::to_vec(&ModuleHelloBody {
8486 manifest: consumer,
8487 protocol_ver: PROTOCOL_VERSION,
8488 control_ops: None,
8489 launch_nonce: None,
8490 })
8491 .expect("HELLO serializes"),
8492 )
8493 .expect("HELLO frame builds");
8494 handler
8495 .handle_control(ConnectionId::new(78), hello)
8496 .expect("consumer registers");
8497 let describe = Frame::build(
8498 FrameType::Request,
8499 control_flags(),
8500 0,
8501 0,
8502 79,
8503 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8504 .expect("request serializes"),
8505 )
8506 .expect("describe frame builds");
8507 let response = handler
8508 .handle_server_describe(describe)
8509 .expect("server.describe succeeds");
8510 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8511 let requirement = &rendered["capability_requirements"][0];
8512 assert_eq!(requirement["consumer"], "consumer");
8513 assert_eq!(requirement["verdict"], "never_provided");
8514 assert_eq!(requirement["episode_seq"], 1);
8515 assert_eq!(requirement["config_satisfiable"], false);
8516 assert_eq!(requirement["runtime_available"], false);
8517 assert!(requirement["detail"]
8518 .as_str()
8519 .expect("detail string")
8520 .contains("credentials-provider/v1"));
8521 }
8522
8523 #[test]
8524 fn catalog_list_omits_capabilities_for_legacy_manifest() {
8525 let registry = Arc::new(Registry::default());
8526 let handler = ControlHandler::new(Arc::clone(®istry));
8527 let hello = handler
8528 .handle_control(
8529 ConnectionId::new(101),
8530 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
8531 )
8532 .expect("legacy HELLO registers");
8533 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
8534
8535 let request = Frame::build(
8536 FrameType::Request,
8537 control_flags(),
8538 0,
8539 0,
8540 102,
8541 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8542 .expect("catalog request serializes"),
8543 )
8544 .expect("catalog request frame builds");
8545 let response = handler
8546 .handle_catalog_list(request, None)
8547 .expect("catalog list succeeds");
8548 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8549 assert!(
8550 body["modules"][0].get("capabilities").is_none(),
8551 "legacy manifest must retain an absent capabilities field on catalog.list"
8552 );
8553 }
8554
8555 #[test]
8556 fn hello_ack_omits_storage_when_no_storage_config() {
8557 let registry = Arc::new(Registry::default());
8558 let handler = ControlHandler::new(Arc::clone(®istry));
8559 let responses = handler
8560 .handle_control(
8561 ConnectionId::new(1),
8562 hello_frame("aft", PROTOCOL_VERSION, 7),
8563 )
8564 .unwrap();
8565 let ack = parse_ack(&responses[0]);
8566 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
8567 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
8568 }
8569
8570 #[tokio::test]
8571 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
8572 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
8573 let registry = Arc::new(Registry::default());
8574 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
8575 let responses = handler
8576 .handle_control(
8577 ConnectionId::new(1),
8578 hello_frame("aft", PROTOCOL_VERSION, 7),
8579 )
8580 .unwrap();
8581 let ack = parse_ack(&responses[0]);
8582 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
8583
8584 let described = handler
8585 .handle_control_frame(
8586 &route_ctx(ConnectionId::new(2)).0,
8587 Frame::build(
8588 FrameType::Request,
8589 control_flags(),
8590 0,
8591 0,
8592 9,
8593 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8594 )
8595 .unwrap(),
8596 )
8597 .await
8598 .unwrap();
8599 let ClientControlResponse::ServerDescribe { machine_id, .. } =
8600 serde_json::from_slice(&described[0].body).unwrap()
8601 else {
8602 panic!("server.describe answered with another shape");
8603 };
8604 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
8605 }
8606
8607 #[test]
8608 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
8609 let registry = Arc::new(Registry::default());
8612 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
8613 crate::daemon_config::StorageConfig::Sqlite {
8614 data_home: std::path::PathBuf::from("/data"),
8615 },
8616 ));
8617
8618 let responses = handler
8619 .handle_control(
8620 ConnectionId::new(1),
8621 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
8622 )
8623 .unwrap();
8624 let ack = parse_ack(&responses[0]);
8625 assert_eq!(
8626 ack.storage,
8627 Some(serde_json::json!({
8628 "module_id": "alfonso-routing",
8629 "storage_namespace": "default",
8630 "isolation": { "kind": "module" },
8631 "backend": {
8632 "backend": "sqlite",
8633 "path": "/data/cortexkit/alfonso-routing/store.db"
8634 }
8635 })),
8636 "the delivered descriptor is the module's own sqlite store path"
8637 );
8638 }
8639
8640 #[test]
8641 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
8642 let registry = Arc::new(Registry::default());
8643 let handler = ControlHandler::new(Arc::clone(®istry));
8644 let conn = ConnectionId::new(1);
8645 let responses = handler
8646 .handle_control(
8647 conn,
8648 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8649 )
8650 .unwrap();
8651 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8652 let registration = registry.get_module("aft").unwrap().unwrap();
8653 assert_eq!(registration.control_ops, module_baseline_control_ops());
8654
8655 let frame =
8656 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
8657 assert!(handler
8658 .guard_module_control_op(&frame, "aft", "route.bind")
8659 .unwrap()
8660 .is_none());
8661 let error = handler
8662 .guard_module_control_op(&frame, "aft", "test.synthetic")
8663 .unwrap()
8664 .expect("synthetic ungranted op should be rejected");
8665 assert_eq!(error.header.ty, FrameType::Error);
8666 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
8667 }
8668
8669 #[test]
8670 fn hello_control_ops_some_adds_optional_grants() {
8671 let registry = Arc::new(Registry::default());
8672 let handler = ControlHandler::new(Arc::clone(®istry));
8673 handler
8674 .handle_control(
8675 ConnectionId::new(1),
8676 hello_frame_with_control_ops(
8677 "aft",
8678 PROTOCOL_VERSION,
8679 7,
8680 Some(vec![
8681 "future.synthetic".to_string(),
8682 "route.bind".to_string(),
8683 ]),
8684 ),
8685 )
8686 .unwrap();
8687 let registration = registry.get_module("aft").unwrap().unwrap();
8688 assert_eq!(
8689 registration.control_ops,
8690 vec![
8691 "route.bind".to_string(),
8692 "route.status".to_string(),
8693 "future.synthetic".to_string(),
8694 ]
8695 );
8696 let frame =
8697 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
8698 assert!(handler
8699 .guard_module_control_op(&frame, "aft", "future.synthetic")
8700 .unwrap()
8701 .is_none());
8702 }
8703
8704 #[tokio::test]
8705 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
8706 let registry = Arc::new(Registry::default());
8707 let forwarding = Arc::new(ForwardingTable::default());
8708 let handler =
8709 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8710 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
8711 hello_via_sink(
8712 &handler,
8713 &module_ctx,
8714 &mut module_rx,
8715 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8716 )
8717 .await;
8718
8719 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
8720 let responses = handler
8721 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
8722 .await
8723 .unwrap();
8724 assert_eq!(responses.len(), 1);
8725 assert_eq!(responses[0].header.ty, FrameType::Error);
8726 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
8727 assert!(module_rx.try_recv().is_err());
8728 }
8729
8730 #[tokio::test]
8731 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
8732 let registry = Arc::new(Registry::default());
8733 let forwarding = Arc::new(ForwardingTable::default());
8734 let handler =
8735 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8736 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
8737 hello_via_sink(
8738 &handler,
8739 &module_ctx,
8740 &mut module_rx,
8741 hello_frame_with_control_ops(
8742 "aft",
8743 PROTOCOL_VERSION,
8744 7,
8745 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8746 ),
8747 )
8748 .await;
8749
8750 let project_root = unique_project_root("demux");
8751 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
8752 let route_handler = handler.clone();
8753 let route_task = tokio::spawn(async move {
8754 route_handler
8755 .handle_control_frame(
8756 &route_client_ctx,
8757 route_open_frame(100, "aft", project_root),
8758 )
8759 .await
8760 .unwrap()
8761 });
8762 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8763 .await
8764 .unwrap()
8765 .unwrap();
8766 assert!(matches!(
8767 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
8768 ModuleControlRequest::RouteBind { .. }
8769 ));
8770
8771 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
8772 let health_handler = handler.clone();
8773 let health_task = tokio::spawn(async move {
8774 health_handler
8775 .handle_control_frame(
8776 &health_client_ctx,
8777 supervisor_health_probe_frame(101, "aft"),
8778 )
8779 .await
8780 .unwrap()
8781 });
8782 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8783 .await
8784 .unwrap()
8785 .unwrap();
8786 assert_eq!(
8787 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8788 ModuleControlRequest::HealthCheck {}
8789 );
8790
8791 handler
8792 .handle_control_frame(
8793 &module_ctx,
8794 health_response(health_frame.header.corr, HealthStatus::Degraded),
8795 )
8796 .await
8797 .unwrap();
8798 let health_response = health_task.await.unwrap();
8799 assert_eq!(health_response.len(), 1);
8800 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8801 ClientControlResponse::SupervisorHealthProbe {
8802 module_id,
8803 status,
8804 detail,
8805 metrics,
8806 } => {
8807 assert_eq!(module_id, "aft");
8808 assert_eq!(status, HealthStatus::Degraded);
8809 assert_eq!(detail.as_deref(), Some("warming"));
8810 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
8811 }
8812 other => panic!("unexpected health response: {other:?}"),
8813 }
8814
8815 handler
8816 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8817 .await
8818 .unwrap();
8819 let route_response = route_task.await.unwrap();
8820 assert!(route_response.is_empty());
8821 let published = route_client_rx.recv().await.unwrap();
8822 assert!(matches!(
8823 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8824 ClientControlResponse::RouteOpen { .. }
8825 ));
8826 }
8827
8828 async fn relay_route_open(
8833 handler: &ControlHandler,
8834 client_connection: ConnectionId,
8835 client_egress: &FrameSink,
8836 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8837 corr: u64,
8838 module_id: &str,
8839 project_root_label: &str,
8840 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
8841 let ctx = RouteCtx {
8842 connection_id: client_connection,
8843 egress: client_egress.clone(),
8844 };
8845 let handler = handler.clone();
8846 let project_root = unique_project_root(project_root_label);
8847 let module_id = module_id.to_string();
8848 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
8849 let task = tokio::spawn(async move {
8850 let _guard = tracing::dispatcher::set_default(&dispatch);
8851 handler
8852 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8853 .await
8854 .unwrap()
8855 });
8856 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8857 .await
8858 .expect("module receives the relayed route.bind")
8859 .expect("module egress is open");
8860 (task, bind.frame)
8861 }
8862
8863 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8864 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8865 ModuleControlRequest::RouteBind {
8866 route_channel,
8867 epoch,
8868 ..
8869 } => (route_channel, epoch),
8870 other => panic!("expected a route.bind request, got {other:?}"),
8871 }
8872 }
8873
8874 fn published_route(frame: &Frame) -> (u16, u32) {
8875 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
8876 ClientControlResponse::RouteOpen {
8877 route_channel,
8878 route_epoch,
8879 } => (route_channel, route_epoch),
8880 other => panic!("expected a route.open response, got {other:?}"),
8881 }
8882 }
8883
8884 #[tokio::test]
8904 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
8905 let registry = Arc::new(Registry::default());
8906 let forwarding = Arc::new(ForwardingTable::default());
8907 let handler =
8908 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8909
8910 let module_connection = ConnectionId::new(30);
8911 let (module_ctx, mut module_rx) = route_ctx(module_connection);
8912 hello_via_sink(
8913 &handler,
8914 &module_ctx,
8915 &mut module_rx,
8916 hello_frame("aft", PROTOCOL_VERSION, 7),
8917 )
8918 .await;
8919
8920 let dying_client = ConnectionId::new(31);
8921 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
8922
8923 let (first_task, first_bind) = relay_route_open(
8926 &handler,
8927 dying_client,
8928 &dying_ctx.egress,
8929 &mut module_rx,
8930 100,
8931 "aft",
8932 "closing-first",
8933 )
8934 .await;
8935 handler
8936 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
8937 .await
8938 .unwrap();
8939 assert!(first_task.await.unwrap().is_empty());
8940 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
8941
8942 let (second_task, second_bind) = relay_route_open(
8944 &handler,
8945 dying_client,
8946 &dying_ctx.egress,
8947 &mut module_rx,
8948 101,
8949 "aft",
8950 "closing-second",
8951 )
8952 .await;
8953 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
8954
8955 assert!(forwarding
8958 .escalate_client_delivery_failure(
8959 dying_client,
8960 first_channel,
8961 first_epoch,
8962 CloseReason::new(
8963 "module_to_client_delivery_failed",
8964 "client egress refused a module frame",
8965 ),
8966 crate::forwarding::UndeliveredFrame {
8967 module_id: None,
8968 sink: &dying_ctx.egress,
8969 },
8970 )
8971 .unwrap());
8972 assert!(!dying_ctx.egress.is_closed());
8973
8974 let ack = handler
8976 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
8977 .await;
8978 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
8979 if module_loop_error.is_some() {
8980 handler.cleanup_connection(module_connection).unwrap();
8984 }
8985 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8989 .await
8990 .ok()
8991 .flatten();
8992
8993 assert!(
8995 registry
8996 .get_module_by_connection(module_connection)
8997 .unwrap()
8998 .is_some(),
8999 "one client's closing connection ended the shared module connection: \
9000 {module_loop_error:?}"
9001 );
9002 let cotenant = ConnectionId::new(32);
9004 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9005 let (cotenant_task, cotenant_bind) = relay_route_open(
9006 &handler,
9007 cotenant,
9008 &cotenant_ctx.egress,
9009 &mut module_rx,
9010 102,
9011 "aft",
9012 "closing-cotenant",
9013 )
9014 .await;
9015 handler
9016 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9017 .await
9018 .unwrap();
9019 assert!(cotenant_task.await.unwrap().is_empty());
9020 let (cotenant_channel, cotenant_epoch) =
9021 published_route(&cotenant_rx.recv().await.unwrap());
9022 assert!(matches!(
9023 forwarding
9024 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9025 .unwrap(),
9026 DataRoute::Client(DataRouteState::Bound(_))
9027 ));
9028
9029 let goodbye = post_ack_module_frame
9032 .expect("module receives a GOODBYE for the abandoned route channel");
9033 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9034 assert_eq!(goodbye.header.channel, abandoned_channel);
9035 assert_eq!(goodbye.header.epoch, abandoned_epoch);
9036
9037 assert!(dying_rx.try_recv().is_err());
9041 let second_response = second_task.await.unwrap();
9042 assert_eq!(second_response.len(), 1);
9043 assert_eq!(
9044 parse_error(&second_response[0])["code"],
9045 "target_unavailable"
9046 );
9047 }
9048
9049 async fn route_open_held_after_bind_send() -> (
9053 ControlHandler,
9054 Arc<tokio::sync::Barrier>,
9055 RouteCtx,
9056 mpsc::Receiver<crate::router::OutboundFrame>,
9057 ) {
9058 let hold = Arc::new(tokio::sync::Barrier::new(2));
9059 let handler = ControlHandler::with_forwarding(
9060 Arc::new(Registry::default()),
9061 Arc::new(ForwardingTable::default()),
9062 )
9063 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
9064 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9065 hello_via_sink(
9066 &handler,
9067 &module_ctx,
9068 &mut module_rx,
9069 hello_frame("aft", PROTOCOL_VERSION, 7),
9070 )
9071 .await;
9072 (handler, hold, module_ctx, module_rx)
9073 }
9074
9075 #[tokio::test]
9082 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
9083 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9084 let client = ConnectionId::new(41);
9085 let (client_ctx, mut client_rx) = route_ctx(client);
9086
9087 let (open_task, bind) = relay_route_open(
9088 &handler,
9089 client,
9090 &client_ctx.egress,
9091 &mut module_rx,
9092 200,
9093 "aft",
9094 "acked-before-mark",
9095 )
9096 .await;
9097 let (module_channel, module_epoch) = route_bind_channel(&bind);
9098 hold.wait().await;
9100
9101 assert!(handler
9103 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9104 .await
9105 .unwrap()
9106 .is_empty());
9107 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
9108
9109 hold.wait().await;
9110 assert!(
9111 open_task.await.unwrap().is_empty(),
9112 "route.open answers accepted"
9113 );
9114
9115 match module_rx.try_recv() {
9118 Err(mpsc::error::TryRecvError::Empty) => {}
9119 Ok(outbound) => panic!(
9120 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
9121 outbound.frame.header.ty,
9122 outbound.frame.header.channel,
9123 outbound.frame.header.epoch
9124 ),
9125 Err(err) => panic!("module egress closed: {err}"),
9126 }
9127
9128 let router = Router::with_control_handler(Arc::new(handler.clone()));
9131 let request = Frame::build(
9132 FrameType::Request,
9133 Flags::new(true, Priority::Interactive, false),
9134 client_channel,
9135 client_epoch,
9136 201,
9137 b"first request".to_vec(),
9138 )
9139 .unwrap();
9140 router
9141 .route_for_connection(&client_ctx, request)
9142 .await
9143 .unwrap();
9144 let delivered = module_rx.try_recv().unwrap().frame;
9145 assert_eq!(delivered.header.ty, FrameType::Request);
9146 assert_eq!(
9147 (delivered.header.channel, delivered.header.epoch),
9148 (module_channel, module_epoch)
9149 );
9150 assert_eq!(delivered.body, b"first request");
9151 }
9152
9153 #[tokio::test]
9159 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
9160 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9161 let client = ConnectionId::new(42);
9162 let (client_ctx, _client_rx) = route_ctx(client);
9163
9164 let (open_task, bind) = relay_route_open(
9165 &handler,
9166 client,
9167 &client_ctx.egress,
9168 &mut module_rx,
9169 300,
9170 "aft",
9171 "released-before-mark",
9172 )
9173 .await;
9174 let (module_channel, module_epoch) = route_bind_channel(&bind);
9175 hold.wait().await;
9176
9177 handler.cleanup_connection(client).unwrap();
9178 assert!(
9179 module_rx.try_recv().is_err(),
9180 "client teardown must not GOODBYE a bind route.open is still queueing"
9181 );
9182
9183 hold.wait().await;
9184 let response = open_task.await.unwrap();
9185 assert_eq!(response.len(), 1);
9186 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9187
9188 let goodbye = module_rx
9189 .try_recv()
9190 .expect("module is told to drop the binding the daemon abandoned")
9191 .frame;
9192 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9193 assert_eq!(
9194 (goodbye.header.channel, goodbye.header.epoch),
9195 (module_channel, module_epoch)
9196 );
9197 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9198 }
9199
9200 #[tokio::test]
9205 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
9206 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9207 let client = ConnectionId::new(43);
9208 let (client_ctx, client_rx) = route_ctx(client);
9209
9210 let (open_task, bind) = relay_route_open(
9211 &handler,
9212 client,
9213 &client_ctx.egress,
9214 &mut module_rx,
9215 400,
9216 "aft",
9217 "gone-client-before-mark",
9218 )
9219 .await;
9220 let (module_channel, module_epoch) = route_bind_channel(&bind);
9221 hold.wait().await;
9222
9223 drop(client_rx);
9224 assert!(client_ctx.egress.is_closed());
9225 handler
9226 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9227 .await
9228 .unwrap();
9229
9230 hold.wait().await;
9231 let response = open_task.await.unwrap();
9232 assert_eq!(response.len(), 1);
9233 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9234
9235 let goodbye = module_rx
9236 .try_recv()
9237 .expect("module is told to drop the binding it acked")
9238 .frame;
9239 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9240 assert_eq!(
9241 (goodbye.header.channel, goodbye.header.epoch),
9242 (module_channel, module_epoch)
9243 );
9244 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9245 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
9246 }
9247
9248 #[test]
9255 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
9256 let handler = ControlHandler::default();
9257 let module_connection = ConnectionId::new(30);
9258 let client_connection = ConnectionId::new(31);
9259
9260 handler
9261 .refuse_to_end_module_connection_for_a_client(
9262 module_connection,
9263 77,
9264 ForwardingError::ConnectionClosing {
9265 connection_id: client_connection,
9266 },
9267 )
9268 .expect("a closing client must never end the module connection");
9269
9270 assert!(matches!(
9271 handler.refuse_to_end_module_connection_for_a_client(
9272 module_connection,
9273 78,
9274 ForwardingError::ConnectionClosing {
9275 connection_id: module_connection,
9276 },
9277 ),
9278 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
9279 connection_id
9280 })) if connection_id == module_connection
9281 ));
9282 assert!(matches!(
9283 handler.refuse_to_end_module_connection_for_a_client(
9284 module_connection,
9285 79,
9286 ForwardingError::Poisoned,
9287 ),
9288 Err(RouterError::Forwarding(ForwardingError::Poisoned))
9289 ));
9290 assert!(matches!(
9291 handler.refuse_to_end_module_connection_for_a_client(
9292 module_connection,
9293 80,
9294 ForwardingError::StaleModuleEndpoint,
9295 ),
9296 Err(RouterError::Forwarding(
9297 ForwardingError::StaleModuleEndpoint
9298 ))
9299 ));
9300 }
9301
9302 #[tokio::test]
9315 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
9316 let registry = Arc::new(Registry::default());
9317 let forwarding = Arc::new(ForwardingTable::default());
9318 let supervisor = SupervisorHandle::new();
9319 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9320 let handler =
9321 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9322 .with_supervisor(supervisor);
9323
9324 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9325 hello_via_sink(
9326 &handler,
9327 &target_ctx,
9328 &mut target_rx,
9329 hello_frame("target", PROTOCOL_VERSION, 1),
9330 )
9331 .await;
9332
9333 let wrong_nonce = handler
9337 .handle_control_frame(
9338 &route_ctx(ConnectionId::new(91)).0,
9339 route_open_frame_with_admission_facts(
9340 20,
9341 "target",
9342 unique_project_root("admission-facts"),
9343 Some(subc_control::ConsumerIdentity {
9344 module_id: "fed".to_string(),
9345 launch_nonce: "not-the-real-nonce".to_string(),
9346 }),
9347 None,
9348 ),
9349 )
9350 .await
9351 .unwrap();
9352 assert_eq!(
9353 parse_error(&wrong_nonce[0])["code"],
9354 "bad_consumer_identity",
9355 "a mismatched launch nonce must be refused, not stamped Reserved"
9356 );
9357
9358 let never_spawned = handler
9362 .handle_control_frame(
9363 &route_ctx(ConnectionId::new(92)).0,
9364 route_open_frame_with_admission_facts(
9365 21,
9366 "target",
9367 unique_project_root("admission-facts"),
9368 Some(subc_control::ConsumerIdentity {
9369 module_id: "never-spawned".to_string(),
9370 launch_nonce: "any-nonce".to_string(),
9371 }),
9372 None,
9373 ),
9374 )
9375 .await
9376 .unwrap();
9377 assert_eq!(
9378 parse_error(&never_spawned[0])["code"],
9379 "bad_consumer_identity",
9380 "an unspawned module_id must be refused rather than accepted for lack of a record"
9381 );
9382 }
9383
9384 #[tokio::test]
9407 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
9408 let registry = Arc::new(Registry::default());
9409 let forwarding = Arc::new(ForwardingTable::default());
9410 let supervisor = SupervisorHandle::new();
9411 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9412 let handler =
9413 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9414 .with_supervisor(supervisor);
9415
9416 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
9417 hello_via_sink(
9418 &handler,
9419 &target_ctx,
9420 &mut target_rx,
9421 hello_frame("target", PROTOCOL_VERSION, 1),
9422 )
9423 .await;
9424
9425 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
9426 let route_handler = handler.clone();
9427 let route_task = tokio::spawn(async move {
9428 route_handler
9429 .handle_control_frame(
9430 &client_ctx,
9431 route_open_frame_with_admission_facts(
9432 30,
9433 "target",
9434 unique_project_root("admission-facts"),
9435 Some(subc_control::ConsumerIdentity {
9436 module_id: "fed".to_string(),
9437 launch_nonce: "fed-nonce".to_string(),
9438 }),
9439 None,
9440 ),
9441 )
9442 .await
9443 .unwrap()
9444 });
9445
9446 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9453 .await
9454 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
9455 .expect("module control channel closed before route.bind");
9456 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9457 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9458 panic!("expected route.bind")
9459 };
9460 assert_eq!(
9461 principal,
9462 Some(Principal::Reserved {
9463 module_id: "fed".to_string()
9464 }),
9465 "a correctly attested consumer must be stamped Reserved for its own id"
9466 );
9467
9468 handler
9469 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9470 .await
9471 .unwrap();
9472 assert!(route_task.await.unwrap().is_empty());
9473 assert!(
9474 matches!(
9475 serde_json::from_slice::<ClientControlResponse>(
9476 &client_rx.recv().await.unwrap().body
9477 )
9478 .unwrap(),
9479 ClientControlResponse::RouteOpen { .. }
9480 ),
9481 "the route must actually open, not merely avoid an error"
9482 );
9483 }
9484
9485 #[tokio::test(start_paused = true)]
9486 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
9487 let registry = Arc::new(Registry::default());
9488 let forwarding = Arc::new(ForwardingTable::default());
9489 let supervisor = SupervisorHandle::new();
9490 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9491 let handler =
9492 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9493 .with_supervisor(supervisor);
9494
9495 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
9496 hello_via_sink(
9497 &handler,
9498 &target_ctx,
9499 &mut target_rx,
9500 hello_frame("target", PROTOCOL_VERSION, 1),
9501 )
9502 .await;
9503
9504 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
9505 let direct_handler = handler.clone();
9506 let direct_open = tokio::spawn(async move {
9507 direct_handler
9508 .handle_control_frame(
9509 &direct_ctx,
9510 route_open_frame(2, "target", unique_project_root("route-census-direct")),
9511 )
9512 .await
9513 .unwrap()
9514 });
9515 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9516 .await
9517 .expect("no direct route.bind within 5s")
9518 .expect("target control channel closed before direct route.bind");
9519 handler
9520 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
9521 .await
9522 .unwrap();
9523 assert!(direct_open.await.unwrap().is_empty());
9524 let _ = direct_rx.recv().await.unwrap();
9525
9526 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
9527 let reserved_handler = handler.clone();
9528 let reserved_open = tokio::spawn(async move {
9529 reserved_handler
9530 .handle_control_frame(
9531 &reserved_ctx,
9532 route_open_frame_with_admission_facts(
9533 3,
9534 "target",
9535 unique_project_root("admission-facts"),
9536 Some(ConsumerIdentity {
9537 module_id: "fed".to_string(),
9538 launch_nonce: "fed-nonce".to_string(),
9539 }),
9540 None,
9541 ),
9542 )
9543 .await
9544 .unwrap()
9545 });
9546 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9547 .await
9548 .expect("no reserved route.bind within 5s")
9549 .expect("target control channel closed before reserved route.bind");
9550 handler
9551 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
9552 .await
9553 .unwrap();
9554 assert!(reserved_open.await.unwrap().is_empty());
9555 let _ = reserved_rx.recv().await.unwrap();
9556
9557 forwarding
9558 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
9559 .unwrap();
9560 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
9561 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
9562 module_id: Some("target".to_string()),
9563 })
9564 .unwrap();
9565 let census_frame =
9566 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
9567 let response = handler
9568 .handle_control_frame(&census_ctx, census_frame)
9569 .await
9570 .unwrap()
9571 .pop()
9572 .unwrap();
9573 let actual: Value = serde_json::from_slice(&response.body).unwrap();
9574 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
9575 assert!(matches!(
9576 decoded,
9577 ClientControlResponse::SupervisorRoutes { .. }
9578 ));
9579 let routes = actual["modules"][0]["routes"].as_array().unwrap();
9580 assert_eq!(routes.len(), 2);
9581 assert!(routes.iter().all(|route| route["draining"] == true));
9582 assert!(
9585 routes.iter().all(|route| route["drain_reason"] == "reload"),
9586 "draining routes must name the drain's reason: {routes:?}"
9587 );
9588 assert!(routes.iter().any(|route| {
9589 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
9590 }));
9591 assert!(routes.iter().any(|route| {
9592 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
9593 }));
9594
9595 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9596 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
9597 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9598 std::fs::write(
9599 &golden_path,
9600 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
9601 )
9602 .unwrap();
9603 }
9604 let expected: Value =
9605 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
9606 assert_eq!(actual, expected);
9607 }
9608
9609 async fn query_live_roots(
9610 handler: &ControlHandler,
9611 module_ctx: &RouteCtx,
9612 ) -> ModuleControlResponseToModule {
9613 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
9614 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
9615 let response = handler
9616 .handle_control_frame(module_ctx, frame)
9617 .await
9618 .unwrap()
9619 .pop()
9620 .unwrap();
9621 serde_json::from_slice(&response.body).unwrap()
9622 }
9623
9624 #[tokio::test(start_paused = true)]
9625 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
9626 let registry = Arc::new(Registry::default());
9627 let forwarding = Arc::new(ForwardingTable::default());
9628 let handler = ControlHandler::with_forwarding(registry, forwarding);
9629 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
9630 hello_via_sink(
9631 &handler,
9632 &target_ctx,
9633 &mut target_rx,
9634 hello_frame("target", PROTOCOL_VERSION, 1),
9635 )
9636 .await;
9637 let root = unique_project_root("live-roots-known");
9638 let path = ProjectRootId::from_path_allowing_missing(root.path())
9639 .unwrap()
9640 .as_path()
9641 .to_path_buf();
9642 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
9643 let open_handler = handler.clone();
9644 let opened = tokio::spawn(async move {
9645 open_handler
9646 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
9647 .await
9648 .unwrap()
9649 });
9650 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9651 .await
9652 .unwrap()
9653 .unwrap();
9654 handler
9655 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
9656 .await
9657 .unwrap();
9658 assert!(opened.await.unwrap().is_empty());
9659 let _ = client_rx.recv().await.unwrap();
9660
9661 let root = unique_project_root("live-roots-pending");
9662 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
9663 .unwrap()
9664 .as_path()
9665 .to_path_buf();
9666 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
9667 let open_handler = handler.clone();
9668 let pending = tokio::spawn(async move {
9669 open_handler
9670 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
9671 .await
9672 .unwrap()
9673 });
9674 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9675 .await
9676 .unwrap()
9677 .unwrap();
9678 let actual = query_live_roots(&handler, &target_ctx).await;
9679 let ModuleControlResponseToModule::LiveRoots {
9680 roots,
9681 unknown_root_bindings,
9682 total_bindings,
9683 } = actual
9684 else {
9685 panic!("expected live roots")
9686 };
9687 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
9688 assert_eq!(unknown_root_bindings, 0);
9689 assert_eq!(
9690 roots.len(),
9691 2,
9692 "root-known arm must retain each canonical root"
9693 );
9694 assert_eq!(
9695 total_bindings,
9696 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9697 );
9698 let counts = roots
9699 .iter()
9700 .map(|root| (root.project_root.clone(), root.bound, root.pending))
9701 .collect::<Vec<_>>();
9702 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
9703 expected.sort_by(|a, b| a.0.cmp(&b.0));
9704 assert_eq!(
9705 counts, expected,
9706 "roots must sort by path and count pending separately"
9707 );
9708 handler
9709 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
9710 .await
9711 .unwrap();
9712 assert!(pending.await.unwrap().is_empty());
9713 }
9714
9715 #[tokio::test(start_paused = true)]
9716 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
9717 let forwarding = Arc::new(ForwardingTable::default());
9718 let handler =
9719 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9720 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
9721 hello_via_sink(
9722 &handler,
9723 &target_ctx,
9724 &mut target_rx,
9725 hello_frame("target", PROTOCOL_VERSION, 1),
9726 )
9727 .await;
9728 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
9729 let pending = forwarding
9730 .begin_route_bind_relay_for_test(
9731 client_ctx.connection_id,
9732 client_ctx.egress.clone(),
9733 2,
9734 "target",
9735 )
9736 .unwrap();
9737 forwarding
9738 .complete_pending_relay(
9739 target_ctx.connection_id,
9740 pending.corr,
9741 RouteBindRelayOutcome::Accepted,
9742 )
9743 .unwrap();
9744 let actual = query_live_roots(&handler, &target_ctx).await;
9745 let ModuleControlResponseToModule::LiveRoots {
9746 roots,
9747 unknown_root_bindings,
9748 total_bindings,
9749 } = actual
9750 else {
9751 panic!("expected live roots")
9752 };
9753 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
9754 assert_eq!(
9755 unknown_root_bindings, 1,
9756 "unknown-root arm must not read as no bindings"
9757 );
9758 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
9759 assert_eq!(
9760 total_bindings,
9761 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9762 );
9763 }
9764
9765 #[tokio::test(start_paused = true)]
9773 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
9774 let forwarding = Arc::new(ForwardingTable::default());
9775 let handler =
9776 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
9777 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
9778 let replies = handler
9779 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
9780 .await
9781 .unwrap();
9782 let queued_by_hello = module_rx.len();
9783
9784 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
9785 let open_handler = handler.clone();
9786 let open = tokio::spawn(async move {
9787 open_handler
9788 .handle_control_frame(
9789 &client_ctx,
9790 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
9791 )
9792 .await
9793 .unwrap()
9794 });
9795 let mut spins = 0;
9797 while module_rx.len() == queued_by_hello {
9798 spins += 1;
9799 assert!(spins < 10_000, "route.open never queued a route.bind");
9800 tokio::task::yield_now().await;
9801 }
9802
9803 for reply in replies {
9805 module_ctx.egress.send(reply).await.unwrap();
9806 }
9807
9808 let first = module_rx.recv().await.unwrap().frame;
9809 assert_eq!(
9810 first.header.ty,
9811 FrameType::HelloAck,
9812 "the first frame a registering module reads must be its HELLO_ACK"
9813 );
9814 assert_eq!(first.header.corr, 7);
9815 let second = module_rx.recv().await.unwrap().frame;
9816 assert_eq!(second.header.ty, FrameType::Request);
9817 assert!(
9818 matches!(
9819 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
9820 ModuleControlRequest::RouteBind { .. }
9821 ),
9822 "the route.bind follows the ack"
9823 );
9824 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
9825
9826 handler
9827 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
9828 .await
9829 .unwrap();
9830 assert!(open.await.unwrap().is_empty());
9831 let _ = client_rx.recv().await.unwrap();
9832 }
9833
9834 #[tokio::test(start_paused = true)]
9835 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
9836 let handler = ControlHandler::with_forwarding(
9837 Arc::new(Registry::default()),
9838 Arc::new(ForwardingTable::default()),
9839 );
9840 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
9841 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
9842 hello_via_sink(
9843 &handler,
9844 &first_ctx,
9845 &mut first_rx,
9846 hello_frame("first", PROTOCOL_VERSION, 1),
9847 )
9848 .await;
9849 hello_via_sink(
9850 &handler,
9851 &second_ctx,
9852 &mut second_rx,
9853 hello_frame("second", PROTOCOL_VERSION, 2),
9854 )
9855 .await;
9856 let root = unique_project_root("second-only");
9857 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
9858 let cloned = handler.clone();
9859 let open = tokio::spawn(async move {
9860 cloned
9861 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
9862 .await
9863 .unwrap()
9864 });
9865 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
9866 .await
9867 .unwrap()
9868 .unwrap();
9869 let first = query_live_roots(&handler, &first_ctx).await;
9870 let second = query_live_roots(&handler, &second_ctx).await;
9871 assert!(
9872 matches!(
9873 first,
9874 ModuleControlResponseToModule::LiveRoots {
9875 total_bindings: 0,
9876 ..
9877 }
9878 ),
9879 "cross-module scope must not expose another module's roots"
9880 );
9881 assert!(
9882 matches!(
9883 second,
9884 ModuleControlResponseToModule::LiveRoots {
9885 total_bindings: 1,
9886 ..
9887 }
9888 ),
9889 "second module must see its pending route"
9890 );
9891 handler
9892 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
9893 .await
9894 .unwrap();
9895 assert!(open.await.unwrap().is_empty());
9896 }
9897
9898 #[tokio::test(start_paused = true)]
9899 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
9900 let handler = ControlHandler::with_forwarding(
9901 Arc::new(Registry::default()),
9902 Arc::new(ForwardingTable::default()),
9903 );
9904 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
9905 hello_via_sink(
9906 &handler,
9907 &target_ctx,
9908 &mut target_rx,
9909 hello_frame("target", PROTOCOL_VERSION, 1),
9910 )
9911 .await;
9912 let actual = query_live_roots(&handler, &target_ctx).await;
9913 let ModuleControlResponseToModule::LiveRoots {
9914 roots,
9915 unknown_root_bindings,
9916 total_bindings,
9917 } = actual
9918 else {
9919 panic!("expected live roots")
9920 };
9921 assert!(roots.is_empty());
9922 assert_eq!(unknown_root_bindings, 0);
9923 assert_eq!(total_bindings, 0);
9924 assert_eq!(
9925 total_bindings,
9926 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
9927 );
9928 }
9929
9930 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
9937 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9938 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
9939 let text = std::fs::read_to_string(&path)
9940 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
9941 let vectors: Vec<(String, Value)> = text
9942 .lines()
9943 .filter(|line| !line.trim().is_empty())
9944 .map(|line| {
9945 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
9946 let id = entry["corpus_id"]
9947 .as_str()
9948 .expect("every vector carries a corpus_id")
9949 .to_string();
9950 (id, entry["package"].clone())
9951 })
9952 .collect();
9953 assert_eq!(
9957 vectors.len(),
9958 3,
9959 "vendored fed corpus changed size; re-sync from subc-federation"
9960 );
9961
9962 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
9977 let richest = vectors
9978 .iter()
9979 .filter_map(|(_, package)| package.as_object())
9980 .map(|object| {
9981 object
9982 .keys()
9983 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
9984 .count()
9985 })
9986 .max()
9987 .unwrap_or(0);
9988 assert!(
9989 richest >= 2,
9990 "vendored corpus no longer carries a package with unmodelled fields, \
9991 so the relay test can no longer distinguish a verbatim relay from a lossy one"
9992 );
9993
9994 vectors
9995 }
9996
9997 #[tokio::test]
10005 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
10006 for (corpus_id, package) in fed_admission_facts_vectors() {
10007 let registry = Arc::new(Registry::default());
10008 let forwarding = Arc::new(ForwardingTable::default());
10009 let supervisor = SupervisorHandle::new();
10010 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10011 let handler =
10012 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10013 .with_supervisor(supervisor)
10014 .with_admission_facts_config(
10015 Some("fed".to_string()),
10016 Some(vec!["target".to_string()]),
10017 );
10018
10019 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10020 hello_via_sink(
10021 &handler,
10022 &target_ctx,
10023 &mut target_rx,
10024 hello_frame("target", PROTOCOL_VERSION, 1),
10025 )
10026 .await;
10027
10028 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
10029 let route_handler = handler.clone();
10030 let expected = package.clone();
10031 let route_task = tokio::spawn(async move {
10032 route_handler
10033 .handle_control_frame(
10034 &client_ctx,
10035 route_open_frame_with_admission_facts(
10036 20,
10037 "target",
10038 unique_project_root("admission-facts"),
10039 Some(subc_control::ConsumerIdentity {
10040 module_id: "fed".to_string(),
10041 launch_nonce: "fed-nonce".to_string(),
10042 }),
10043 Some(package),
10044 ),
10045 )
10046 .await
10047 .unwrap()
10048 });
10049
10050 let bind_frame = target_rx.recv().await.unwrap();
10051 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10052 let ModuleControlRequest::RouteBind {
10053 admission_facts, ..
10054 } = bind
10055 else {
10056 panic!("{corpus_id}: expected route.bind")
10057 };
10058 assert_eq!(
10059 admission_facts,
10060 Some(expected),
10061 "{corpus_id}: relay must not add, drop or reshape any field"
10062 );
10063
10064 handler
10065 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10066 .await
10067 .unwrap();
10068 route_task.await.unwrap();
10069 }
10070 }
10071
10072 #[tokio::test]
10073 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
10074 let registry = Arc::new(Registry::default());
10075 let forwarding = Arc::new(ForwardingTable::default());
10076 let supervisor = SupervisorHandle::new();
10077 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10078 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
10079 let handler =
10080 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10081 .with_supervisor(supervisor)
10082 .with_admission_facts_config(
10083 Some("fed".to_string()),
10084 Some(vec!["target".to_string()]),
10085 );
10086
10087 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
10088 hello_via_sink(
10089 &handler,
10090 &target_ctx,
10091 &mut target_rx,
10092 hello_frame("target", PROTOCOL_VERSION, 1),
10093 )
10094 .await;
10095 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
10096 hello_via_sink(
10097 &handler,
10098 &other_ctx,
10099 &mut other_rx,
10100 hello_frame("other", PROTOCOL_VERSION, 2),
10101 )
10102 .await;
10103
10104 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
10105 let expected_facts = facts.clone();
10106 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
10107 let route_handler = handler.clone();
10108 let route_task = tokio::spawn(async move {
10109 route_handler
10110 .handle_control_frame(
10111 &client_ctx,
10112 route_open_frame_with_admission_facts(
10113 10,
10114 "target",
10115 unique_project_root("admission-facts"),
10116 Some(subc_control::ConsumerIdentity {
10117 module_id: "fed".to_string(),
10118 launch_nonce: "fed-nonce".to_string(),
10119 }),
10120 Some(facts.clone()),
10121 ),
10122 )
10123 .await
10124 .unwrap()
10125 });
10126 let bind_frame = target_rx.recv().await.unwrap();
10127 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10128 let ModuleControlRequest::RouteBind {
10129 admission_facts, ..
10130 } = bind
10131 else {
10132 panic!("expected route.bind")
10133 };
10134 assert_eq!(admission_facts, Some(expected_facts));
10135 handler
10136 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10137 .await
10138 .unwrap();
10139 assert!(route_task.await.unwrap().is_empty());
10140 assert!(matches!(
10141 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
10142 .unwrap(),
10143 ClientControlResponse::RouteOpen { .. }
10144 ));
10145
10146 let direct = handler
10147 .handle_control_frame(
10148 &route_ctx(ConnectionId::new(73)).0,
10149 route_open_frame_with_admission_facts(
10150 11,
10151 "target",
10152 unique_project_root("admission-facts"),
10153 None,
10154 Some(json!({"x": 1})),
10155 ),
10156 )
10157 .await
10158 .unwrap();
10159 assert_eq!(
10160 parse_error(&direct[0])["code"],
10161 "admission_facts_not_permitted"
10162 );
10163
10164 let different_reserved = handler
10165 .handle_control_frame(
10166 &route_ctx(ConnectionId::new(77)).0,
10167 route_open_frame_with_admission_facts(
10168 15,
10169 "target",
10170 unique_project_root("admission-facts"),
10171 Some(subc_control::ConsumerIdentity {
10172 module_id: "other".to_string(),
10173 launch_nonce: "other-nonce".to_string(),
10174 }),
10175 Some(json!({"x": 1})),
10176 ),
10177 )
10178 .await
10179 .unwrap();
10180 assert_eq!(
10181 parse_error(&different_reserved[0])["code"],
10182 "admission_facts_not_permitted"
10183 );
10184
10185 let other_target = handler
10186 .handle_control_frame(
10187 &route_ctx(ConnectionId::new(74)).0,
10188 route_open_frame_with_admission_facts(
10189 12,
10190 "other",
10191 unique_project_root("admission-facts"),
10192 Some(subc_control::ConsumerIdentity {
10193 module_id: "fed".to_string(),
10194 launch_nonce: "fed-nonce".to_string(),
10195 }),
10196 Some(json!({"x": 1})),
10197 ),
10198 )
10199 .await
10200 .unwrap();
10201 assert_eq!(
10202 parse_error(&other_target[0])["code"],
10203 "admission_facts_target_not_allowed"
10204 );
10205
10206 let nonexistent = handler
10207 .handle_control_frame(
10208 &route_ctx(ConnectionId::new(75)).0,
10209 route_open_frame_with_admission_facts(
10210 13,
10211 "missing",
10212 unique_project_root("admission-facts"),
10213 None,
10214 Some(json!({"x": 1})),
10215 ),
10216 )
10217 .await
10218 .unwrap();
10219 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
10220
10221 let described = handler
10222 .handle_control_frame(
10223 &route_ctx(ConnectionId::new(76)).0,
10224 Frame::build(
10225 FrameType::Request,
10226 control_flags(),
10227 0,
10228 0,
10229 14,
10230 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10231 )
10232 .unwrap(),
10233 )
10234 .await
10235 .unwrap();
10236 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10237 serde_json::from_slice(&described[0].body).unwrap()
10238 else {
10239 panic!("expected server.describe response")
10240 };
10241 assert!(capabilities
10242 .iter()
10243 .any(|cap| cap == "admission_facts_relay_v1"));
10244 }
10245
10246 #[tokio::test]
10247 async fn admission_facts_without_configured_carrier_are_rejected() {
10248 let registry = Arc::new(Registry::default());
10249 let forwarding = Arc::new(ForwardingTable::default());
10250 let handler = ControlHandler::with_forwarding(registry, forwarding);
10251 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
10252 hello_via_sink(
10253 &handler,
10254 &target_ctx,
10255 &mut target_rx,
10256 hello_frame("target", PROTOCOL_VERSION, 1),
10257 )
10258 .await;
10259
10260 let responses = handler
10261 .handle_control_frame(
10262 &route_ctx(ConnectionId::new(79)).0,
10263 route_open_frame_with_admission_facts(
10264 16,
10265 "target",
10266 unique_project_root("admission-facts"),
10267 None,
10268 Some(json!({"x": 1})),
10269 ),
10270 )
10271 .await
10272 .unwrap();
10273 assert_eq!(
10274 parse_error(&responses[0])["code"],
10275 "admission_facts_not_permitted"
10276 );
10277 }
10278
10279 #[tokio::test]
10280 async fn route_open_relays_consumer_capabilities_verbatim() {
10281 let registry = Arc::new(Registry::default());
10282 let forwarding = Arc::new(ForwardingTable::default());
10283 let handler =
10284 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10285 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
10286 hello_via_sink(
10287 &handler,
10288 &module_ctx,
10289 &mut module_rx,
10290 hello_frame("aft", PROTOCOL_VERSION, 7),
10291 )
10292 .await;
10293
10294 let expected = vec!["elicitation".to_string(), "roots".to_string()];
10295 let expected_for_request = expected.clone();
10296 let project_root = unique_project_root("consumer-capabilities-present");
10297 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
10298 let route_handler = handler.clone();
10299 let route_task = tokio::spawn(async move {
10300 route_handler
10301 .handle_control_frame(
10302 &client_ctx,
10303 route_open_frame_with_consumer_capabilities(
10304 401,
10305 "aft",
10306 project_root,
10307 Some(expected_for_request),
10308 ),
10309 )
10310 .await
10311 .unwrap()
10312 });
10313 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10314 .await
10315 .unwrap()
10316 .unwrap();
10317 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10318 let ModuleControlRequest::RouteBind {
10319 consumer_capabilities,
10320 ..
10321 } = bind
10322 else {
10323 panic!("expected route.bind request, got {bind:?}");
10324 };
10325 assert_eq!(consumer_capabilities, Some(expected.clone()));
10326
10327 handler
10328 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10329 .await
10330 .unwrap();
10331 let route_response = route_task.await.unwrap();
10332 assert!(route_response.is_empty());
10333 let published = client_rx.recv().await.unwrap();
10334 assert!(matches!(
10335 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10336 ClientControlResponse::RouteOpen { .. }
10337 ));
10338 }
10339
10340 #[tokio::test]
10341 async fn route_open_without_consumer_capabilities_relays_none() {
10342 let registry = Arc::new(Registry::default());
10343 let forwarding = Arc::new(ForwardingTable::default());
10344 let handler =
10345 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10346 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
10347 hello_via_sink(
10348 &handler,
10349 &module_ctx,
10350 &mut module_rx,
10351 hello_frame("aft", PROTOCOL_VERSION, 7),
10352 )
10353 .await;
10354
10355 let project_root = unique_project_root("consumer-capabilities-absent");
10356 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
10357 let route_handler = handler.clone();
10358 let route_task = tokio::spawn(async move {
10359 route_handler
10360 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
10361 .await
10362 .unwrap()
10363 });
10364 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10365 .await
10366 .unwrap()
10367 .unwrap();
10368 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10369 let ModuleControlRequest::RouteBind {
10370 consumer_capabilities,
10371 ..
10372 } = bind
10373 else {
10374 panic!("expected route.bind request, got {bind:?}");
10375 };
10376 assert_eq!(consumer_capabilities, None);
10377
10378 handler
10379 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10380 .await
10381 .unwrap();
10382 let route_response = route_task.await.unwrap();
10383 assert!(route_response.is_empty());
10384 let published = client_rx.recv().await.unwrap();
10385 assert!(matches!(
10386 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10387 ClientControlResponse::RouteOpen { .. }
10388 ));
10389 }
10390
10391 async fn bind_role_versions_for(
10395 sent: Option<BTreeMap<String, String>>,
10396 connection: u64,
10397 ) -> Option<BTreeMap<String, String>> {
10398 let registry = Arc::new(Registry::default());
10399 let forwarding = Arc::new(ForwardingTable::default());
10400 let handler =
10401 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10402 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
10403 hello_via_sink(
10404 &handler,
10405 &module_ctx,
10406 &mut module_rx,
10407 hello_frame("aft", PROTOCOL_VERSION, 7),
10408 )
10409 .await;
10410 let project_root = unique_project_root("role-versions");
10411 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
10412 let route_handler = handler.clone();
10413 let route_task = tokio::spawn(async move {
10414 route_handler
10415 .handle_control_frame(
10416 &client_ctx,
10417 route_open_frame_with_role_versions(403, "aft", project_root, sent),
10418 )
10419 .await
10420 .unwrap()
10421 });
10422 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10423 .await
10424 .expect("a well-formed route.open reaches the module as a bind")
10425 .unwrap();
10426 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10427 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
10428 panic!("expected route.bind request, got {bind:?}");
10429 };
10430 handler
10431 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10432 .await
10433 .unwrap();
10434 assert!(route_task.await.unwrap().is_empty());
10435 let published = client_rx.recv().await.unwrap();
10436 assert!(matches!(
10437 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10438 ClientControlResponse::RouteOpen { .. }
10439 ));
10440 role_versions
10441 }
10442
10443 #[tokio::test]
10444 async fn route_open_relays_role_versions_verbatim() {
10445 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
10446 assert_eq!(
10447 bind_role_versions_for(Some(sent.clone()), 141).await,
10448 Some(sent)
10449 );
10450 }
10451
10452 #[tokio::test]
10455 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
10456 assert_eq!(bind_role_versions_for(None, 143).await, None);
10457 assert_eq!(
10458 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
10459 None
10460 );
10461 }
10462
10463 #[tokio::test]
10464 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
10465 let registry = Arc::new(Registry::default());
10466 let forwarding = Arc::new(ForwardingTable::default());
10467 let handler =
10468 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10469 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
10470 hello_via_sink(
10471 &handler,
10472 &module_ctx,
10473 &mut module_rx,
10474 hello_frame("aft", PROTOCOL_VERSION, 7),
10475 )
10476 .await;
10477
10478 let nine: BTreeMap<String, String> = (0..9)
10479 .map(|index| (format!("role-{index}"), "v1".to_string()))
10480 .collect();
10481 for (label, malformed) in [
10482 (
10483 "invalid role name",
10484 role_versions(&[("Tool_Provider", "v1")]),
10485 ),
10486 ("invalid version", role_versions(&[("tool-provider", "v0")])),
10487 ("nine entries", nine),
10488 ] {
10489 let responses = tokio::time::timeout(
10492 Duration::from_secs(1),
10493 handler.handle_control_frame(
10494 &route_ctx(ConnectionId::new(148)).0,
10495 route_open_frame_with_role_versions(
10496 404,
10497 "aft",
10498 unique_project_root("role-versions-malformed"),
10499 Some(malformed),
10500 ),
10501 ),
10502 )
10503 .await
10504 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
10505 .unwrap();
10506 assert_eq!(responses.len(), 1, "{label}");
10507 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
10508 let error = parse_error(&responses[0]);
10509 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
10510 assert_eq!(
10511 error["detail"]["field"], "role_versions",
10512 "{label}: {error}"
10513 );
10514 assert!(
10515 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
10516 "{label}: a malformed declaration is terminal"
10517 );
10518 assert!(
10519 module_rx.try_recv().is_err(),
10520 "{label}: the module must never see a bind"
10521 );
10522 }
10523 }
10524
10525 #[tokio::test]
10529 async fn route_role_versions_capability_is_advertised() {
10530 let handler = ControlHandler::new(Arc::new(Registry::default()));
10531 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
10532 let ack = hello_via_sink(
10533 &handler,
10534 &ctx,
10535 &mut rx,
10536 hello_frame("m", PROTOCOL_VERSION, 1),
10537 )
10538 .await;
10539 let ack = parse_ack(&ack);
10540 assert!(
10541 ack.subc_capabilities
10542 .iter()
10543 .any(|c| c == "route-role-versions/v1"),
10544 "{:?}",
10545 ack.subc_capabilities
10546 );
10547
10548 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
10549 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
10550 let reply = handler
10551 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
10552 .await
10553 .unwrap()
10554 .pop()
10555 .unwrap();
10556 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10557 serde_json::from_slice(&reply.body).unwrap()
10558 else {
10559 panic!("not a server.describe reply");
10560 };
10561 assert!(
10562 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
10563 "{capabilities:?}"
10564 );
10565 }
10566
10567 #[tokio::test]
10571 async fn catalog_events_capability_is_advertised() {
10572 let handler = ControlHandler::new(Arc::new(Registry::default()));
10573 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
10574 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
10575 let reply = handler
10576 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
10577 .await
10578 .unwrap()
10579 .pop()
10580 .unwrap();
10581 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10582 serde_json::from_slice(&reply.body).unwrap()
10583 else {
10584 panic!("not a server.describe reply");
10585 };
10586 assert!(
10587 capabilities
10588 .iter()
10589 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
10590 "{capabilities:?}"
10591 );
10592 }
10593
10594 #[tokio::test]
10595 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
10596 let registry = Arc::new(Registry::default());
10597 let forwarding = Arc::new(ForwardingTable::default());
10598 let handler =
10599 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10600 .with_health_probe_timeout(Duration::from_secs(5));
10601 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
10602 hello_via_sink(
10603 &handler,
10604 &module_ctx,
10605 &mut module_rx,
10606 non_routable_hello_frame_with_control_ops(
10607 "mcp",
10608 300,
10609 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10610 ),
10611 )
10612 .await;
10613 assert!(registry
10614 .get_module("mcp")
10615 .unwrap()
10616 .unwrap()
10617 .manifest
10618 .provides
10619 .is_empty());
10620
10621 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
10622 let route_response = handler
10623 .handle_control_frame(
10624 &route_client_ctx,
10625 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
10626 )
10627 .await
10628 .unwrap();
10629 assert_eq!(route_response[0].header.ty, FrameType::Error);
10630 assert_eq!(
10631 parse_error(&route_response[0])["code"],
10632 "target_unavailable"
10633 );
10634 assert!(parse_error(&route_response[0])["message"]
10635 .as_str()
10636 .unwrap()
10637 .contains("does not provide the requested target"));
10638 assert!(module_rx.try_recv().is_err());
10639
10640 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
10641 let health_handler = handler.clone();
10642 let health_task = tokio::spawn(async move {
10643 health_handler
10644 .handle_control_frame(
10645 &health_client_ctx,
10646 supervisor_health_probe_frame(302, "mcp"),
10647 )
10648 .await
10649 .unwrap()
10650 });
10651 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10652 .await
10653 .unwrap()
10654 .unwrap();
10655 assert_eq!(
10656 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10657 ModuleControlRequest::HealthCheck {}
10658 );
10659 handler
10660 .handle_control_frame(
10661 &module_ctx,
10662 health_response(health_frame.header.corr, HealthStatus::Ok),
10663 )
10664 .await
10665 .unwrap();
10666 let health_response = health_task.await.unwrap();
10667 assert_eq!(health_response[0].header.ty, FrameType::Response);
10668 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10669 ClientControlResponse::SupervisorHealthProbe {
10670 module_id, status, ..
10671 } => {
10672 assert_eq!(module_id, "mcp");
10673 assert_eq!(status, HealthStatus::Ok);
10674 }
10675 other => panic!("unexpected health response: {other:?}"),
10676 }
10677
10678 forwarding
10683 .cleanup_connection(module_ctx.connection_id)
10684 .unwrap();
10685 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
10686 let cleanup_response = tokio::time::timeout(
10687 Duration::from_millis(200),
10688 handler.handle_control_frame(
10689 &cleanup_probe_ctx,
10690 supervisor_health_probe_frame(303, "mcp"),
10691 ),
10692 )
10693 .await
10694 .expect("probe should fail immediately when the control lane is gone")
10695 .unwrap();
10696 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
10697 assert_eq!(
10698 parse_error(&cleanup_response[0])["code"],
10699 "target_unavailable"
10700 );
10701 assert!(parse_error(&cleanup_response[0])["message"]
10702 .as_str()
10703 .unwrap()
10704 .contains("no module connection"));
10705
10706 handler
10707 .cleanup_connection(module_ctx.connection_id)
10708 .unwrap();
10709 }
10710
10711 #[tokio::test]
10712 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
10713 let registry = Arc::new(Registry::default());
10714 let supervisor_handle = SupervisorHandle::new();
10715 let supervisor =
10716 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
10717 .with_handle(supervisor_handle.clone())
10718 .with_connection_file_path(
10719 std::env::temp_dir()
10720 .join(format!("subc-route-open-warming-{}", std::process::id())),
10721 );
10722 let module = supervisor
10723 .supervise_configured(
10724 ModuleSpec {
10725 module_id: "warming".to_string(),
10726 program: fake_aft_stub_path(),
10727 args: Vec::new(),
10728 env: Vec::new(),
10729 reserved: false,
10730 reserved_prefixes: Vec::new(),
10731 protocol: ModuleProtocol::Subc,
10732 overlap: Default::default(),
10733 },
10734 true,
10735 )
10736 .unwrap();
10737 assert_eq!(module.state().unwrap(), ModuleState::Running);
10738
10739 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
10740 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
10741 let response = handler
10742 .handle_control_frame(
10743 &ctx,
10744 route_open_frame(304, "warming", unique_project_root("warming")),
10745 )
10746 .await
10747 .unwrap();
10748 module.stop().await.unwrap();
10749
10750 assert_eq!(response[0].header.ty, FrameType::Error);
10751 let error = parse_error(&response[0]);
10752 assert_eq!(error["code"], "module_warming");
10753 assert!(error["message"]
10754 .as_str()
10755 .unwrap()
10756 .contains("state=running, enabled=true, live=false"));
10757 }
10758
10759 #[test]
10760 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
10761 let handler = ControlHandler::new(Arc::new(Registry::default()));
10762 let capture = EventCapture::default();
10763 let _subscriber =
10764 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10765 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
10766 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
10767 let pending = (0..limit).collect::<Vec<_>>();
10768 let response = handler
10769 .route_open_capacity_refusal(
10770 &ctx,
10771 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
10772 "busy",
10773 pending.len(),
10774 limit,
10775 )
10776 .unwrap();
10777 assert_eq!(parse_error(&response)["code"], "target_unavailable");
10778 let event = capture
10779 .events()
10780 .into_iter()
10781 .find(|event| {
10782 event.target == "control"
10783 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
10784 })
10785 .expect("connection admission refusal event");
10786 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10787 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10788 }
10789
10790 #[test]
10791 fn route_open_target_cap_logs_admission_reason_and_capacity() {
10792 let handler = ControlHandler::new(Arc::new(Registry::default()));
10793 let capture = EventCapture::default();
10794 let _subscriber =
10795 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10796 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
10797 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
10798 let guards = (0..limit)
10799 .map(|_| {
10800 handler
10801 .route_bind_concurrency
10802 .try_admit("busy", limit)
10803 .unwrap()
10804 })
10805 .collect::<Vec<_>>();
10806 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
10807 Err(in_flight) => in_flight,
10808 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
10809 };
10810 let response = handler
10811 .route_open_target_capacity_refusal(
10812 &ctx,
10813 &route_open_frame(397, "busy", unique_project_root("target-cap")),
10814 "busy",
10815 in_flight,
10816 )
10817 .unwrap();
10818 assert_eq!(parse_error(&response)["code"], "target_unavailable");
10819 let event = capture
10820 .events()
10821 .into_iter()
10822 .find(|event| {
10823 event.target == "control"
10824 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
10825 })
10826 .expect("target admission refusal event");
10827 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
10828 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
10829 drop(guards);
10830 }
10831
10832 #[tokio::test(flavor = "current_thread")]
10833 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
10834 let handler = ControlHandler::new(Arc::new(Registry::default()));
10835 let capture = EventCapture::default();
10836 let _subscriber =
10837 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10838 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
10839 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10840
10841 let _ = handler
10842 .handle_client_control_request(
10843 &ctx,
10844 frame,
10845 ClientControlRequest::SupervisorSetEnabled {
10846 module_id: "broca".to_string(),
10847 enabled: false,
10848 },
10849 )
10850 .await
10851 .unwrap();
10852
10853 let events = capture
10854 .events()
10855 .into_iter()
10856 .filter(|event| {
10857 event.target == "control"
10858 && event.fields.get("message").map(String::as_str)
10859 == Some("supervisor request received")
10860 })
10861 .collect::<Vec<_>>();
10862 assert_eq!(events.len(), 1, "one supervisor request log line");
10863 let event = &events[0];
10864 assert_eq!(event.level, tracing::Level::INFO);
10865 assert_eq!(
10866 event.fields.get("op"),
10867 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
10868 );
10869 assert_eq!(
10870 event.fields.get("module_id"),
10871 Some(&"\"broca\"".to_string())
10872 );
10873 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
10874 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
10875 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
10876 }
10877
10878 #[tokio::test(flavor = "current_thread")]
10879 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
10880 let registry = Arc::new(Registry::default());
10881 let module_connection = ConnectionId::new(303);
10882 registry
10883 .register_with_control_ops(
10884 manifest("aft", PROTOCOL_VERSION),
10885 PROTOCOL_VERSION,
10886 module_connection,
10887 module_baseline_control_ops(),
10888 )
10889 .unwrap();
10890 let handler = ControlHandler::new(registry);
10891 let capture = EventCapture::default();
10892 let _subscriber =
10893 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10894 let (ctx, _rx) = route_ctx(module_connection);
10895 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
10896
10897 let _ = handler
10898 .handle_client_control_request(
10899 &ctx,
10900 frame,
10901 ClientControlRequest::SupervisorSetEnabled {
10902 module_id: "broca".to_string(),
10903 enabled: false,
10904 },
10905 )
10906 .await
10907 .unwrap();
10908
10909 let callers = capture
10910 .events()
10911 .into_iter()
10912 .filter(|event| {
10913 event.fields.get("message").map(String::as_str)
10914 == Some("supervisor request received")
10915 })
10916 .map(|event| event.fields.get("caller").cloned())
10917 .collect::<Vec<_>>();
10918 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
10919 }
10920
10921 #[tokio::test(flavor = "current_thread")]
10922 async fn supervisor_list_does_not_log_request_received() {
10923 let handler = ControlHandler::new(Arc::new(Registry::default()));
10924 let capture = EventCapture::default();
10925 let _subscriber =
10926 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10927 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
10928 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
10929
10930 handler
10931 .handle_client_control_request(&ctx, frame, ClientControlRequest::SupervisorList {})
10932 .await
10933 .unwrap();
10934
10935 assert!(capture.events().into_iter().all(|event| {
10936 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10937 }));
10938 }
10939
10940 #[tokio::test(flavor = "current_thread")]
10941 async fn supervisor_rescan_preview_does_not_log_request_received() {
10942 let handler = ControlHandler::new(Arc::new(Registry::default()));
10943 let capture = EventCapture::default();
10944 let _subscriber =
10945 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10946 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
10947 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
10948
10949 handler
10950 .handle_client_control_request(
10951 &ctx,
10952 frame,
10953 ClientControlRequest::SupervisorRescan { preview: true },
10954 )
10955 .await
10956 .unwrap();
10957
10958 assert!(capture.events().into_iter().all(|event| {
10959 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
10960 }));
10961 }
10962
10963 #[tokio::test]
10968 async fn route_open_refusal_names_the_check_that_refused() {
10969 let handler = ControlHandler::new(Arc::new(Registry::default()));
10970 let capture = EventCapture::default();
10971 let _subscriber =
10972 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10973 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
10974 let response = handler
10975 .handle_control_frame(
10976 &ctx,
10977 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
10978 )
10979 .await
10980 .unwrap();
10981
10982 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
10983 let event = capture
10984 .events()
10985 .into_iter()
10986 .find(|event| {
10987 event.target == "control"
10988 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
10989 })
10990 .expect("route.open refusal event");
10991 assert_eq!(
10992 event.fields.get("reason"),
10993 Some(&"\"not_registered\"".to_string())
10994 );
10995 }
10996
10997 #[tokio::test]
10998 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
10999 let registry = Arc::new(Registry::default());
11000 let supervisor_handle = SupervisorHandle::new();
11001 let supervisor =
11002 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11003 .with_handle(supervisor_handle.clone())
11004 .with_connection_file_path(std::env::temp_dir().join(format!(
11005 "subc-route-open-refusal-info-{}",
11006 std::process::id()
11007 )));
11008 let module = supervisor
11009 .supervise_configured(
11010 ModuleSpec {
11011 module_id: "warming".to_string(),
11012 program: fake_aft_stub_path(),
11013 args: Vec::new(),
11014 env: Vec::new(),
11015 reserved: false,
11016 reserved_prefixes: Vec::new(),
11017 protocol: ModuleProtocol::Subc,
11018 overlap: Default::default(),
11019 },
11020 true,
11021 )
11022 .unwrap();
11023 assert_eq!(module.state().unwrap(), ModuleState::Running);
11024
11025 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11026 assert!(handler
11027 .counters()
11028 .snapshot()
11029 .get("route_open_refused_by_code")
11030 .is_none());
11031 let capture = EventCapture::default();
11032 let _subscriber =
11033 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11034 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
11035 let response = handler
11036 .handle_control_frame(
11037 &ctx,
11038 route_open_frame(394, "warming", unique_project_root("refusal-info")),
11039 )
11040 .await
11041 .unwrap();
11042 module.stop().await.unwrap();
11043
11044 assert_eq!(parse_error(&response[0])["code"], "module_warming");
11045 let event = capture
11046 .events()
11047 .into_iter()
11048 .find(|event| {
11049 event.target == "control"
11050 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
11051 })
11052 .expect("route.open refusal event");
11053 assert_eq!(
11054 event.fields.get("module_id"),
11055 Some(&"\"warming\"".to_string())
11056 );
11057 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
11058 assert_eq!(
11059 event.fields.get("reason"),
11060 Some(&"\"supervised_not_registered\"".to_string())
11061 );
11062 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
11063 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
11064 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
11065 assert_eq!(
11066 handler.counters().snapshot()["route_open_refused_by_code"],
11067 json!({ "module_warming": 1 })
11068 );
11069 }
11070
11071 const OUTAGE_START: &str = "route.open refusing module: not serving";
11072 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
11073
11074 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
11075 capture
11076 .events()
11077 .into_iter()
11078 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
11079 .collect()
11080 }
11081
11082 fn supervise_stub(
11083 registry: &Arc<Registry>,
11084 module_id: &str,
11085 enabled: bool,
11086 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
11087 let supervisor_handle = SupervisorHandle::new();
11088 let supervisor =
11089 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
11090 .with_handle(supervisor_handle.clone())
11091 .with_connection_file_path(std::env::temp_dir().join(format!(
11092 "subc-route-outage-{module_id}-{}",
11093 std::process::id()
11094 )));
11095 let module = supervisor
11096 .supervise_configured(
11097 ModuleSpec {
11098 module_id: module_id.to_string(),
11099 program: fake_aft_stub_path(),
11100 args: Vec::new(),
11101 env: Vec::new(),
11102 reserved: false,
11103 reserved_prefixes: Vec::new(),
11104 protocol: ModuleProtocol::Subc,
11105 overlap: Default::default(),
11106 },
11107 enabled,
11108 )
11109 .unwrap();
11110 (supervisor_handle, module)
11111 }
11112
11113 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
11114 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
11115 module_id: module_id.to_string(),
11116 drain_timeout_ms: Some(50),
11117 })
11118 .unwrap();
11119 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11120 }
11121
11122 #[test]
11126 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
11127 let registry = Arc::new(Registry::default());
11128 let forwarding = Arc::new(ForwardingTable::default());
11129 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11130 let second = ControlHandler::with_forwarding(registry, forwarding);
11131 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
11132 }
11133
11134 #[tokio::test(flavor = "current_thread")]
11138 async fn route_open_unknown_module_refusals_add_no_outage_state() {
11139 let handler = ControlHandler::new(Arc::new(Registry::default()));
11140 let capture = EventCapture::default();
11141 let _subscriber =
11142 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11143 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
11144 for corr in 0..8 {
11145 let response = handler
11146 .handle_control_frame(
11147 &ctx,
11148 route_open_frame(
11149 380 + corr,
11150 &format!("nobody-{corr}"),
11151 unique_project_root("outage-unknown"),
11152 ),
11153 )
11154 .await
11155 .unwrap();
11156 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11157 }
11158
11159 assert_eq!(handler.route_outages.tracked_module_count(), 0);
11160 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
11161 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
11162 }
11163
11164 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11169 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
11170 let registry = Arc::new(Registry::default());
11171 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
11172 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11173 let capture = EventCapture::default();
11174 let _subscriber =
11175 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11176 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
11177 handler.route_outages.record_accepted("outage-restart");
11180
11181 let response = handler
11182 .handle_control_frame(
11183 &ctx,
11184 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
11185 )
11186 .await
11187 .unwrap();
11188 assert_eq!(response[0].header.ty, FrameType::Error);
11189 let starts = outage_lines(&capture, OUTAGE_START);
11190 assert_eq!(starts.len(), 1, "{starts:?}");
11191 assert_eq!(starts[0].level, tracing::Level::WARN);
11192 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
11193 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
11194 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
11195 handler.route_outages.record_accepted("outage-restart");
11196 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
11197
11198 let restart = handler
11199 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
11200 .await
11201 .unwrap();
11202 assert_eq!(
11203 restart[0].header.ty,
11204 FrameType::Response,
11205 "{:?}",
11206 parse_error(&restart[0])
11207 );
11208 handler
11209 .handle_control_frame(
11210 &ctx,
11211 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
11212 )
11213 .await
11214 .unwrap();
11215 module.stop().await.unwrap();
11216
11217 let starts = outage_lines(&capture, OUTAGE_START);
11218 assert_eq!(starts.len(), 2, "{starts:?}");
11219 assert_eq!(starts[1].level, tracing::Level::INFO);
11220 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
11221 }
11222
11223 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11227 async fn failed_operator_restart_leaves_no_operator_mark() {
11228 let registry = Arc::new(Registry::default());
11229 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
11230 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11231 let capture = EventCapture::default();
11232 let _subscriber =
11233 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11234 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
11235 handler.route_outages.record_accepted("outage-disabled");
11236
11237 let restart = handler
11238 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
11239 .await
11240 .unwrap();
11241 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
11242 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
11243
11244 handler
11245 .handle_control_frame(
11246 &ctx,
11247 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
11248 )
11249 .await
11250 .unwrap();
11251 let starts = outage_lines(&capture, OUTAGE_START);
11252 assert_eq!(starts.len(), 1, "{starts:?}");
11253 assert_eq!(starts[0].level, tracing::Level::WARN);
11254 }
11255
11256 #[tokio::test(flavor = "current_thread")]
11257 async fn route_open_unknown_module_escapes_target_module_id() {
11258 let handler = ControlHandler::new(Arc::new(Registry::default()));
11259 let capture = EventCapture::default();
11260 let _subscriber =
11261 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11262 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
11263 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11264 let response = handler
11265 .handle_control_frame(
11266 &ctx,
11267 route_open_frame(
11268 395,
11269 hostile_module_id,
11270 unique_project_root("hostile-target-module-id"),
11271 ),
11272 )
11273 .await
11274 .unwrap();
11275
11276 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11277 let event = capture
11278 .events()
11279 .into_iter()
11280 .find(|event| {
11281 event.target == "control"
11282 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11283 })
11284 .expect("route.open unknown-module refusal event");
11285 let logged = event.fields.get("module_id").expect("module_id field");
11286 assert!(!logged.bytes().any(|byte| byte < 0x20));
11287 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
11288 }
11289
11290 #[tokio::test(flavor = "current_thread")]
11291 async fn route_open_module_rejection_uses_daemon_counter_key() {
11292 let registry = Arc::new(Registry::default());
11293 let forwarding = Arc::new(ForwardingTable::default());
11294 let handler =
11295 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11296 let module_connection = ConnectionId::new(95);
11297 let (module_ctx, mut module_rx) = route_ctx(module_connection);
11298 hello_via_sink(
11299 &handler,
11300 &module_ctx,
11301 &mut module_rx,
11302 hello_frame("aft", PROTOCOL_VERSION, 395),
11303 )
11304 .await;
11305
11306 let client_connection = ConnectionId::new(96);
11307 let (client_ctx, _client_rx) = route_ctx(client_connection);
11308 let capture = EventCapture::default();
11309 let _subscriber =
11310 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11311 let (route_task, bind) = relay_route_open(
11312 &handler,
11313 client_connection,
11314 &client_ctx.egress,
11315 &mut module_rx,
11316 396,
11317 "aft",
11318 "hostile-module-code",
11319 )
11320 .await;
11321 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
11322 let rejection = Frame::build(
11323 FrameType::Error,
11324 control_flags(),
11325 0,
11326 0,
11327 bind.header.corr,
11328 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
11329 )
11330 .unwrap();
11331 handler
11332 .handle_control_frame(&module_ctx, rejection)
11333 .await
11334 .unwrap();
11335
11336 let response = route_task.await.unwrap();
11337 assert_eq!(parse_error(&response[0])["code"], hostile_code);
11338 let counters = handler.counters().snapshot();
11339 assert_eq!(
11340 counters["route_open_refused_by_code"],
11341 json!({ "module_rejected": 1 })
11342 );
11343 assert!(counters["route_open_refused_by_code"]
11344 .get(hostile_code)
11345 .is_none());
11346
11347 let event = capture
11348 .events()
11349 .into_iter()
11350 .find(|event| {
11351 event.target == "control"
11352 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
11353 })
11354 .expect("route.open module-rejection refusal event");
11355 let logged = event.fields.get("module_code").expect("module_code field");
11356 assert!(!logged.bytes().any(|byte| byte < 0x20));
11357 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
11358 }
11359
11360 #[tokio::test]
11361 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
11362 let registry = Arc::new(Registry::default());
11363 let supervisor_handle = SupervisorHandle::new();
11364 let missing_program = std::env::temp_dir().join(format!(
11365 "subc-route-open-missing-program-{}",
11366 std::process::id()
11367 ));
11368 let supervisor =
11369 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11370 .with_handle(supervisor_handle.clone());
11371 let module = supervisor
11372 .supervise_configured(
11373 ModuleSpec {
11374 module_id: "failed".to_string(),
11375 program: missing_program,
11376 args: Vec::new(),
11377 env: Vec::new(),
11378 reserved: false,
11379 reserved_prefixes: Vec::new(),
11380 protocol: ModuleProtocol::Subc,
11381 overlap: Default::default(),
11382 },
11383 true,
11384 )
11385 .unwrap();
11386 assert_eq!(module.state().unwrap(), ModuleState::Failed);
11387
11388 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11389 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
11390 let response = handler
11391 .handle_control_frame(
11392 &ctx,
11393 route_open_frame(305, "failed", unique_project_root("failed")),
11394 )
11395 .await
11396 .unwrap();
11397
11398 assert_eq!(response[0].header.ty, FrameType::Error);
11399 let error = parse_error(&response[0]);
11400 assert_eq!(error["code"], "target_unavailable");
11401 assert!(error["message"]
11402 .as_str()
11403 .unwrap()
11404 .contains("state=failed, enabled=true, live=false"));
11405 }
11406
11407 #[tokio::test]
11408 async fn route_open_role_mismatch_remains_target_unavailable() {
11409 let registry = Arc::new(Registry::default());
11410 let handler = ControlHandler::new(Arc::clone(®istry));
11411 handler
11412 .handle_control(
11413 ConnectionId::new(41),
11414 non_routable_hello_frame_with_control_ops("health-only", 306, None),
11415 )
11416 .unwrap();
11417
11418 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
11419 let response = handler
11420 .handle_control_frame(
11421 &ctx,
11422 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
11423 )
11424 .await
11425 .unwrap();
11426
11427 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11428 assert!(parse_error(&response[0])["message"]
11429 .as_str()
11430 .unwrap()
11431 .contains("does not provide the requested target"));
11432 }
11433
11434 #[tokio::test]
11435 async fn route_open_inactive_registration_remains_target_unavailable() {
11436 let registry = Arc::new(Registry::default());
11437 let handler = ControlHandler::new(Arc::clone(®istry));
11438 handler
11439 .handle_control(
11440 ConnectionId::new(43),
11441 hello_frame("inactive", PROTOCOL_VERSION, 308),
11442 )
11443 .unwrap();
11444 assert!(registry
11445 .set_module_state_for_test("inactive", ChannelState::Closed)
11446 .unwrap());
11447
11448 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
11449 let response = handler
11450 .handle_control_frame(
11451 &ctx,
11452 route_open_frame(309, "inactive", unique_project_root("inactive")),
11453 )
11454 .await
11455 .unwrap();
11456
11457 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11458 assert!(parse_error(&response[0])["message"]
11459 .as_str()
11460 .unwrap()
11461 .contains("is not active"));
11462 }
11463
11464 #[tokio::test]
11465 async fn late_health_reply_is_recorded_through_the_module_response_path() {
11466 let registry = Arc::new(Registry::default());
11467 let forwarding = Arc::new(ForwardingTable::default());
11468 let supervisor_handle = SupervisorHandle::new();
11469 let supervisor =
11470 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
11471 .with_forwarding(Arc::clone(&forwarding))
11472 .with_handle(supervisor_handle.clone());
11473 let module = supervisor
11474 .supervise_configured(
11475 crate::ModuleSpec {
11476 module_id: "late-health-response".to_string(),
11477 program: PathBuf::from("disabled-module"),
11478 args: Vec::new(),
11479 env: Vec::new(),
11480 reserved: false,
11481 reserved_prefixes: Vec::new(),
11482 protocol: ModuleProtocol::Subc,
11483 overlap: Default::default(),
11484 },
11485 false,
11486 )
11487 .unwrap();
11488 let handler =
11489 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11490 .with_supervisor(supervisor_handle);
11491 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
11492 handler
11493 .handle_control_frame(
11494 &module_ctx,
11495 hello_frame_with_control_ops(
11496 "late-health-response",
11497 PROTOCOL_VERSION,
11498 7,
11499 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11500 ),
11501 )
11502 .await
11503 .unwrap();
11504 let probe_started_at = Instant::now() - Duration::from_millis(80);
11505 let pending = forwarding
11506 .begin_health_probe_rpc_for(
11507 "late-health-response",
11508 MODULE_CONTROL_OP_HEALTH_CHECK,
11509 probe_started_at,
11510 Instant::now() - Duration::from_millis(1),
11511 )
11512 .unwrap();
11513 assert!(forwarding
11514 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
11515 .unwrap());
11516
11517 let responses = handler
11518 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
11519 .await
11520 .unwrap();
11521
11522 assert!(responses.is_empty());
11523 let health = module.status().unwrap().health;
11524 assert_eq!(health.late_answer_count, 1);
11525 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
11526 }
11527
11528 #[tokio::test]
11529 async fn health_probe_timeout_and_module_death_are_typed() {
11530 let registry = Arc::new(Registry::default());
11531 let forwarding = Arc::new(ForwardingTable::default());
11532 let handler =
11533 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11534 .with_health_probe_timeout(Duration::from_millis(50));
11535 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11536 hello_via_sink(
11537 &handler,
11538 &module_ctx,
11539 &mut module_rx,
11540 hello_frame_with_control_ops(
11541 "aft",
11542 PROTOCOL_VERSION,
11543 7,
11544 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11545 ),
11546 )
11547 .await;
11548
11549 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11550 let responses = handler
11551 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
11552 .await
11553 .unwrap();
11554 assert_eq!(responses[0].header.ty, FrameType::Error);
11555 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
11556 let _ = module_rx.try_recv();
11557
11558 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
11559 let health_handler = handler.clone();
11560 let death_task = tokio::spawn(async move {
11561 health_handler
11562 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
11563 .await
11564 .unwrap()
11565 });
11566 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11567 .await
11568 .unwrap()
11569 .unwrap();
11570 handler
11571 .cleanup_connection(module_ctx.connection_id)
11572 .unwrap();
11573 let responses = death_task.await.unwrap();
11574 assert_eq!(responses[0].header.ty, FrameType::Error);
11575 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
11576 }
11577
11578 #[test]
11579 fn hello_requires_exact_protocol_version() {
11580 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
11581 let registry = Arc::new(Registry::default());
11582 let handler = ControlHandler::new(Arc::clone(®istry));
11583 let responses = handler
11584 .handle_control(
11585 ConnectionId::new(connection),
11586 hello_frame("aft", offered, 9),
11587 )
11588 .unwrap();
11589
11590 assert_eq!(responses.len(), 1);
11591 assert_eq!(responses[0].header.ty, FrameType::Error);
11592 let error = parse_error(&responses[0]);
11593 assert_eq!(error["code"], "version_unsupported");
11594 assert!(registry.get_module("aft").unwrap().is_none());
11595 assert_eq!(registry.active_registration_count().unwrap(), 0);
11596 }
11597 }
11598
11599 #[test]
11600 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
11601 let registry = Arc::new(Registry::default());
11602 let forwarding = Arc::new(ForwardingTable::default());
11603 let handler =
11604 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11605 let module_connection = ConnectionId::new(301);
11606 let registration = registry
11607 .register_with_control_ops(
11608 manifest("aft-push", PROTOCOL_VERSION),
11609 PROTOCOL_VERSION,
11610 module_connection,
11611 module_baseline_control_ops(),
11612 )
11613 .unwrap();
11614 let (module_tx, _module_rx) = mpsc::channel(8);
11615 let endpoint = forwarding
11616 .register_module_connection(
11617 module_connection,
11618 "aft-push".to_string(),
11619 PROTOCOL_VERSION,
11620 manifest_concurrency(®istration.manifest),
11621 FrameSink::new(module_tx),
11622 )
11623 .unwrap();
11624
11625 let unknown = Frame::build(
11627 FrameType::Push,
11628 control_flags(),
11629 0,
11630 0,
11631 5,
11632 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
11633 )
11634 .unwrap();
11635 let out = handler.handle_status_update(endpoint, unknown).unwrap();
11636 assert!(
11637 out.is_empty(),
11638 "unknown push op must be ignored, got {out:?}"
11639 );
11640
11641 let malformed = Frame::build(
11643 FrameType::Push,
11644 control_flags(),
11645 0,
11646 0,
11647 6,
11648 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
11649 )
11650 .unwrap();
11651 let out = handler.handle_status_update(endpoint, malformed).unwrap();
11652 assert_eq!(out.len(), 1);
11653 assert_eq!(out[0].header.ty, FrameType::Error);
11654 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
11655 }
11656
11657 #[test]
11658 fn hello_rejected_when_connection_already_owns_client_routes() {
11659 let registry = Arc::new(Registry::default());
11660 let forwarding = Arc::new(ForwardingTable::default());
11661 let handler =
11662 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11663 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
11665 let client_connection = ConnectionId::new(202);
11666
11667 let responses = handler
11670 .handle_control(
11671 client_connection,
11672 hello_frame("aft-second", PROTOCOL_VERSION, 9),
11673 )
11674 .unwrap();
11675 assert_eq!(responses[0].header.ty, FrameType::Error);
11676 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
11677 assert!(registry.get_module("aft-second").unwrap().is_none());
11678 }
11679
11680 #[tokio::test]
11681 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
11682 let registry = Arc::new(Registry::default());
11683 let forwarding = Arc::new(ForwardingTable::default());
11684 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
11685 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
11686 hello_via_sink(
11687 &handler,
11688 &module_ctx,
11689 &mut module_rx,
11690 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
11691 )
11692 .await;
11693 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
11694 let pending = forwarding
11695 .begin_route_bind_relay_for_test(
11696 client_ctx.connection_id,
11697 client_ctx.egress.clone(),
11698 2,
11699 "alpha",
11700 )
11701 .unwrap();
11702 forwarding
11703 .complete_pending_relay(
11704 module_ctx.connection_id,
11705 pending.corr,
11706 RouteBindRelayOutcome::Accepted,
11707 )
11708 .unwrap();
11709 client_rx.try_recv().unwrap();
11710 for module_id in ["beta", "alpha"] {
11711 let replies = handler
11712 .handle_control_frame(
11713 &module_ctx,
11714 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
11715 )
11716 .await
11717 .unwrap();
11718 assert_eq!(replies.len(), 1, "second HELLO must be refused");
11719 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
11720 }
11721 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
11722 assert!(registry.get_module("beta").unwrap().is_none());
11723 assert!(matches!(
11724 forwarding
11725 .lookup_data_route(
11726 client_ctx.connection_id,
11727 pending.client_channel,
11728 pending.client_epoch,
11729 )
11730 .unwrap(),
11731 DataRoute::Client(DataRouteState::Bound(_))
11732 ));
11733 assert!(handler
11734 .hello_launch_nonces
11735 .lock()
11736 .unwrap()
11737 .presented(module_ctx.connection_id, Some("alpha-nonce")));
11738 assert!(module_rx.try_recv().is_err());
11739 }
11740
11741 #[test]
11742 fn reserved_module_hello_requires_matching_launch_nonce() {
11743 let registry = Arc::new(Registry::default());
11744 let supervisor = SupervisorHandle::new();
11745 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
11748 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11749
11750 let no_nonce = handler
11752 .handle_control(
11753 ConnectionId::new(1),
11754 hello_frame("vault", PROTOCOL_VERSION, 1),
11755 )
11756 .unwrap();
11757 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
11758 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
11759 assert!(registry.get_module("vault").unwrap().is_none());
11760
11761 let wrong = handler
11763 .handle_control(
11764 ConnectionId::new(2),
11765 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
11766 )
11767 .unwrap();
11768 assert_eq!(wrong[0].header.ty, FrameType::Error);
11769 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
11770 assert!(registry.get_module("vault").unwrap().is_none());
11771
11772 let ok = handler
11774 .handle_control(
11775 ConnectionId::new(3),
11776 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
11777 )
11778 .unwrap();
11779 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
11780 assert!(registry.get_module("vault").unwrap().is_some());
11781 }
11782
11783 #[test]
11784 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
11785 let registry = Arc::new(Registry::default());
11786 let supervisor = SupervisorHandle::new();
11787 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11788 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11789 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11790
11791 let squat = handler
11792 .handle_control(
11793 ConnectionId::new(1),
11794 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
11795 )
11796 .unwrap();
11797 assert_eq!(squat[0].header.ty, FrameType::Error);
11798 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
11799 assert!(parse_error(&squat[0])["message"]
11800 .as_str()
11801 .unwrap()
11802 .contains("fed:"));
11803
11804 let accepted_peer = handler
11805 .handle_control(
11806 ConnectionId::new(2),
11807 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
11808 )
11809 .unwrap();
11810 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
11811
11812 let accepted_short = handler
11813 .handle_control(
11814 ConnectionId::new(3),
11815 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
11816 )
11817 .unwrap();
11818 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
11819
11820 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
11821 let response = handler
11822 .handle_control(
11823 ConnectionId::new(conn),
11824 hello_frame(module_id, PROTOCOL_VERSION, conn),
11825 )
11826 .unwrap();
11827 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
11828 }
11829 }
11830
11831 #[test]
11832 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
11833 let registry = Arc::new(Registry::default());
11834 let supervisor = SupervisorHandle::new();
11835 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
11836 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
11837 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
11838 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
11839
11840 let owner_nonce = handler
11841 .handle_control(
11842 ConnectionId::new(1),
11843 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
11844 )
11845 .unwrap();
11846 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
11847 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
11848 assert!(registry.get_module("fed:special").unwrap().is_none());
11849
11850 let exact_nonce = handler
11851 .handle_control(
11852 ConnectionId::new(2),
11853 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
11854 )
11855 .unwrap();
11856 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
11857 assert!(registry.get_module("fed:special").unwrap().is_some());
11858 }
11859
11860 #[test]
11861 fn non_reserved_module_ignores_launch_nonce() {
11862 let registry = Arc::new(Registry::default());
11863 let handler = ControlHandler::new(Arc::clone(®istry));
11866 let no_nonce = handler
11867 .handle_control(
11868 ConnectionId::new(1),
11869 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
11870 )
11871 .unwrap();
11872 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
11873 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
11874
11875 let echoed_nonce = handler
11876 .handle_control(
11877 ConnectionId::new(2),
11878 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
11879 )
11880 .unwrap();
11881 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
11882 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
11883 }
11884
11885 #[test]
11886 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
11887 let handler = ControlHandler::default();
11888 let conn = ConnectionId::new(1);
11889 let malformed = Frame::build(
11890 FrameType::Hello,
11891 control_flags(),
11892 0,
11893 0,
11894 3,
11895 b"{not json".to_vec(),
11896 )
11897 .unwrap();
11898
11899 let error = handler.handle_control(conn, malformed).unwrap();
11900 assert_eq!(error[0].header.ty, FrameType::Error);
11901 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
11902
11903 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
11904 let pong = handler.handle_control(conn, ping).unwrap();
11905 assert_eq!(pong[0].header.ty, FrameType::Pong);
11906 assert_eq!(pong[0].header.corr, 4);
11907 }
11908
11909 #[test]
11910 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
11911 let registry = Arc::new(Registry::default());
11912 let handler = ControlHandler::new(Arc::clone(®istry));
11913
11914 handler
11915 .handle_control(
11916 ConnectionId::new(1),
11917 hello_frame("aft", PROTOCOL_VERSION, 1),
11918 )
11919 .unwrap();
11920 let duplicate = handler
11921 .handle_control(
11922 ConnectionId::new(2),
11923 hello_frame("aft", PROTOCOL_VERSION, 2),
11924 )
11925 .unwrap();
11926
11927 assert_eq!(duplicate[0].header.ty, FrameType::Error);
11928 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11929 let registration = registry.get_module("aft").unwrap().unwrap();
11930 assert_eq!(registration.connection_id, ConnectionId::new(1));
11931 }
11932
11933 #[test]
11934 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
11935 let registry = Arc::new(Registry::default());
11936 let forwarding = Arc::new(ForwardingTable::default());
11937 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
11938 let handler =
11939 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11940 .with_process_liveness(process_liveness);
11941 let (ctx, route_channel, route_epoch) =
11942 bind_liveness_route(®istry, &forwarding, "aft-dead");
11943 let responses = handler
11944 .handle_route_poll(
11945 &ctx,
11946 route_poll_frame(41, PollKind::Liveness, route_channel),
11947 route_channel,
11948 route_epoch,
11949 PollKind::Liveness,
11950 )
11951 .unwrap();
11952
11953 assert_eq!(responses.len(), 1);
11954 assert_eq!(responses[0].header.ty, FrameType::Response);
11955 assert_route_poll_liveness(&responses[0], false);
11956 }
11957
11958 #[test]
11959 fn liveness_poll_without_process_source_uses_bound_route() {
11960 let registry = Arc::new(Registry::default());
11961 let forwarding = Arc::new(ForwardingTable::default());
11962 let handler =
11963 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11964 let (ctx, route_channel, route_epoch) =
11965 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
11966 let responses = handler
11967 .handle_route_poll(
11968 &ctx,
11969 route_poll_frame(42, PollKind::Liveness, route_channel),
11970 route_channel,
11971 route_epoch,
11972 PollKind::Liveness,
11973 )
11974 .unwrap();
11975
11976 assert_route_poll_liveness(&responses[0], true);
11977 }
11978
11979 #[test]
11980 fn liveness_poll_untracked_process_source_uses_bound_route() {
11981 let registry = Arc::new(Registry::default());
11982 let forwarding = Arc::new(ForwardingTable::default());
11983 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
11984 let handler =
11985 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11986 .with_process_liveness(process_liveness);
11987 let (ctx, route_channel, route_epoch) =
11988 bind_liveness_route(®istry, &forwarding, "aft-untracked");
11989 let responses = handler
11990 .handle_route_poll(
11991 &ctx,
11992 route_poll_frame(43, PollKind::Liveness, route_channel),
11993 route_channel,
11994 route_epoch,
11995 PollKind::Liveness,
11996 )
11997 .unwrap();
11998
11999 assert_route_poll_liveness(&responses[0], true);
12000 }
12001
12002 #[tokio::test]
12003 async fn unknown_op_returns_unknown_control_op() {
12004 let handler = ControlHandler::default();
12005 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
12006 let request = Frame::build(
12007 FrameType::Request,
12008 control_flags(),
12009 0,
12010 0,
12011 55,
12012 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
12013 )
12014 .unwrap();
12015
12016 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12017
12018 assert_eq!(response.len(), 1);
12019 assert_eq!(response[0].header.ty, FrameType::Error);
12020 assert_eq!(response[0].header.corr, 55);
12021 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
12022 }
12023
12024 #[tokio::test]
12025 async fn supervisor_provenance_rejects_unknown_exact_module() {
12026 let handler = ControlHandler::default();
12027 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
12028 let request = Frame::build(
12029 FrameType::Request,
12030 control_flags(),
12031 0,
12032 0,
12033 57,
12034 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
12035 )
12036 .unwrap();
12037
12038 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12039
12040 assert_eq!(response.len(), 1);
12041 assert_eq!(response[0].header.ty, FrameType::Error);
12042 assert_eq!(response[0].header.corr, 57);
12043 let error = parse_error(&response[0]);
12044 assert_eq!(error["code"], "unknown_module");
12045 assert_eq!(error["message"], "module_id 'missing' is not supervised");
12046 }
12047
12048 #[test]
12049 fn provenance_probe_override_keeps_handler_tests_deterministic() {
12050 let expected = subc_control::RunningImageAgreement::Unavailable {
12051 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12052 };
12053 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
12054 assert_eq!(handler.provenance_probe_override, Some(expected));
12055 }
12056
12057 #[test]
12058 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
12059 let verdict = reload_verdict(
12060 std::path::Path::new("/bin/new"),
12061 Some(std::path::Path::new("/bin/old")),
12062 subc_control::RunningImageAgreement::Unavailable {
12063 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12064 },
12065 );
12066 assert!(matches!(
12067 verdict.path,
12068 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
12069 if configured == std::path::Path::new("/bin/new")
12070 && spawned_from == std::path::Path::new("/bin/old")
12071 ));
12072 }
12073
12074 #[cfg(windows)]
12075 #[test]
12076 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
12077 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
12078 let configured = dir.join("module.exe");
12079 std::fs::write(&configured, b"executable path fixture").unwrap();
12080 let spawned = std::fs::canonicalize(&configured).unwrap();
12081 assert_ne!(
12082 configured, spawned,
12083 "fixture must exercise different path spellings"
12084 );
12085 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
12086 let lower_case = configured.to_string_lossy().to_lowercase();
12087 for spelling in [
12088 &configured,
12089 Path::new(&forward_slashes),
12090 Path::new(&lower_case),
12091 ] {
12092 let verdict = reload_verdict(
12093 spelling,
12094 Some(&spawned),
12095 subc_control::RunningImageAgreement::Unavailable {
12096 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12097 },
12098 );
12099 assert_eq!(
12100 verdict.path,
12101 ReloadPathAgreement::Match,
12102 "configured {spelling:?}, spawned {spawned:?}"
12103 );
12104 }
12105 }
12106
12107 #[cfg(windows)]
12108 #[test]
12109 fn windows_reload_path_keeps_different_existing_program_pending() {
12110 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
12111 let old = dir.join("old.exe");
12112 let new = dir.join("new.exe");
12113 std::fs::write(&old, b"same executable content").unwrap();
12114 std::fs::copy(&old, &new).unwrap();
12115 let spawned = std::fs::canonicalize(&old).unwrap();
12116 let image = subc_control::RunningImageAgreement::Unavailable {
12117 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12118 };
12119 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
12120 assert!(
12121 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
12122 if configured == new && spawned_from == spawned)
12123 );
12124 let missing = dir.join("missing.exe");
12125 assert!(matches!(
12126 reload_verdict(&missing, Some(&spawned), image).path,
12127 ReloadPathAgreement::Mismatch { .. }
12128 ));
12129 }
12130
12131 #[test]
12132 fn reload_verdict_detects_replaced_image_at_same_path() {
12133 let image = subc_control::RunningImageAgreement::Mismatch {
12134 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
12135 digest: "old".into(),
12136 },
12137 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
12138 digest: "new".into(),
12139 },
12140 };
12141 let verdict = reload_verdict(
12142 std::path::Path::new("/bin/same"),
12143 Some(std::path::Path::new("/bin/same")),
12144 image.clone(),
12145 );
12146 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12147 assert_eq!(verdict.image, image);
12148 }
12149
12150 #[test]
12151 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
12152 let image = subc_control::RunningImageAgreement::Unavailable {
12153 reason: subc_control::RunningImageUnavailableReason::NotRunning,
12154 };
12155 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
12156 assert_eq!(
12157 verdict.path,
12158 subc_control::ReloadPathAgreement::Unavailable {
12159 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
12160 }
12161 );
12162 assert_eq!(verdict.image, image);
12163
12164 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
12165 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12166 };
12167 let verdict = reload_verdict(
12168 std::path::Path::new("/bin/same"),
12169 Some(std::path::Path::new("/bin/same")),
12170 unconfirmed.clone(),
12171 );
12172 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12173 assert_eq!(verdict.image, unconfirmed);
12174 }
12175
12176 #[test]
12177 fn reload_verdict_preserves_each_image_unavailability_reason() {
12178 use subc_control::RunningImageUnavailableReason as Reason;
12179
12180 for reason in [
12181 Reason::NotRunning,
12182 Reason::UnsupportedPlatform,
12183 Reason::RunningExecutableUnreadable,
12184 Reason::SpawnedPathUnreadable,
12185 Reason::HashFailed,
12186 Reason::ProcessIdentityUnconfirmed,
12187 Reason::Unknown("future_probe_reason".to_string()),
12188 ] {
12189 let image = subc_control::RunningImageAgreement::Unavailable {
12190 reason: reason.clone(),
12191 };
12192 let verdict = reload_verdict(
12193 std::path::Path::new("/bin/same"),
12194 Some(std::path::Path::new("/bin/same")),
12195 image.clone(),
12196 );
12197 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12198 assert_eq!(verdict.image, image, "{reason:?}");
12199 }
12200 }
12201
12202 #[tokio::test]
12203 async fn malformed_control_bodies_return_invalid_control_body() {
12204 let handler = ControlHandler::default();
12205 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
12206
12207 for (corr, body) in [
12208 (56, br#"{"route_channel":1}"#.as_slice()),
12209 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
12210 (
12211 58,
12212 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
12213 ),
12214 ] {
12215 let request = Frame::build(
12216 FrameType::Request,
12217 control_flags(),
12218 0,
12219 0,
12220 corr,
12221 body.to_vec(),
12222 )
12223 .unwrap();
12224 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12225
12226 assert_eq!(response.len(), 1);
12227 assert_eq!(response[0].header.ty, FrameType::Error);
12228 assert_eq!(response[0].header.corr, corr);
12229 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
12230 }
12231 }
12232
12233 #[tokio::test]
12234 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
12235 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12236 let registry = Arc::new(Registry::default());
12237 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
12238 let router = Router::with_control_handler(Arc::clone(&control));
12239 let connection = router.begin_connection();
12240 let (ctx, mut rx) = route_ctx(connection.id());
12241
12242 router
12243 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
12244 .await
12245 .unwrap();
12246 let response = rx.recv().await.unwrap();
12247 let ack = parse_ack(&response);
12248 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
12249 let channel = 1;
12250
12251 let goodbye =
12252 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
12253 router.route_for_connection(&ctx, goodbye).await.unwrap();
12254 assert!(rx.try_recv().is_err());
12255 assert!(registry.get_module("aft").unwrap().is_none());
12256
12257 router
12258 .route_for_connection(&ctx, channel_request(channel, 13))
12259 .await
12260 .unwrap();
12261 let error_frame = rx.recv().await.unwrap();
12262 assert_eq!(error_frame.header.ty, FrameType::Error);
12263 assert_eq!(error_frame.header.channel, channel);
12264 let captured = crate::router::test_log::captured_logs(&logs);
12265 assert_eq!(
12266 captured
12267 .lines()
12268 .filter(|line| {
12269 line.contains("module_id=aft")
12270 && line.contains("reason=explicit_goodbye")
12271 && line.contains("module registration ended")
12272 })
12273 .count(),
12274 1,
12275 "unexpected GOODBYE registry log: {captured}"
12276 );
12277 }
12278
12279 #[tokio::test]
12280 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
12281 let registry = Arc::new(Registry::default());
12282 let handler = ControlHandler::new(registry).with_capability_config(
12283 [("prov".to_string(), true), ("cons".to_string(), true)],
12284 BTreeMap::new(),
12285 );
12286 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
12287 register_capability_manifest(
12288 &handler,
12289 &provider_ctx,
12290 &mut provider_rx,
12291 capability_manifest("prov", &["thing/v1"], &[]),
12292 1,
12293 )
12294 .await;
12295 let mut consumer = capability_manifest("cons", &[], &[]);
12296 consumer.capabilities.as_mut().unwrap().requires.push(
12297 subc_protocol::manifest::CapabilityRequirement {
12298 capability: "thing/v1".to_string(),
12299 need: subc_protocol::manifest::CapabilityNeed::Required,
12300 },
12301 );
12302 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
12303 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
12304 assert_eq!(
12305 handler.capability_evaluator.verdict("cons", "thing/v1"),
12306 Some(CapabilityVerdict::Provided)
12307 );
12308 let (mut client_rx, _) = open_route_for_capability_test(
12309 &handler,
12310 &provider_ctx,
12311 &mut provider_rx,
12312 703,
12313 3,
12314 "prov",
12315 None,
12316 )
12317 .await;
12318 let goodbye =
12319 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12320 handler
12321 .handle_control_frame(&provider_ctx, goodbye)
12322 .await
12323 .unwrap();
12324 assert_eq!(
12325 handler.capability_evaluator.verdict("cons", "thing/v1"),
12326 Some(CapabilityVerdict::NeverProvided)
12327 );
12328 let closed = client_rx
12329 .try_recv()
12330 .expect("GOODBYE pushes route.closed before route GOODBYE");
12331 assert!(
12332 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
12333 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
12334 );
12335 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
12336 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
12337 }
12338
12339 struct RouteClosedFixture {
12340 handler: ControlHandler,
12341 #[cfg(unix)]
12342 supervisor: Supervisor,
12343 module_ctx: RouteCtx,
12344 #[cfg(unix)]
12345 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12346 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12347 route: Arc<crate::forwarding::RouteBinding>,
12348 }
12349
12350 async fn route_closed_fixture() -> RouteClosedFixture {
12351 let registry = Arc::new(Registry::default());
12352 let handler = ControlHandler::new(Arc::clone(®istry));
12353 #[cfg(unix)]
12354 let supervisor = Supervisor::new(registry, RestartPolicy::default())
12355 .with_forwarding(Arc::clone(&handler.forwarding));
12356 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
12357 register_capability_manifest(
12358 &handler,
12359 &module_ctx,
12360 &mut module_rx,
12361 capability_manifest("prov", &[], &[]),
12362 1,
12363 )
12364 .await;
12365 let (client_rx, bind) = open_route_for_capability_test(
12366 &handler,
12367 &module_ctx,
12368 &mut module_rx,
12369 702,
12370 2,
12371 "prov",
12372 None,
12373 )
12374 .await;
12375 let ModuleControlRequest::RouteBind {
12376 route_channel: channel,
12377 epoch: route_epoch,
12378 ..
12379 } = bind
12380 else {
12381 panic!("expected route.bind");
12382 };
12383 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
12384 handler
12385 .forwarding
12386 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
12387 .unwrap()
12388 else {
12389 panic!("the acknowledged route must be bound");
12390 };
12391 RouteClosedFixture {
12392 handler,
12393 #[cfg(unix)]
12394 supervisor,
12395 module_ctx,
12396 #[cfg(unix)]
12397 module_rx,
12398 client_rx,
12399 route,
12400 }
12401 }
12402
12403 impl RouteClosedFixture {
12404 #[cfg(unix)]
12405 async fn cut(&mut self) {
12406 let notices = async {
12407 let mut draining = self.module_rx.recv().await.unwrap();
12408 assert!(matches!(
12409 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
12410 &draining.body
12411 )
12412 .unwrap(),
12413 subc_protocol::session::ModuleControlCommand::Draining {
12414 reason: RouteCloseReason::Restart,
12415 ..
12416 }
12417 ));
12418 draining.flushed.take().unwrap().send(()).unwrap();
12419 let mut closing = self.client_rx.recv().await.unwrap();
12420 assert!(matches!(
12421 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
12422 ClientControlPush::RouteClosing {
12423 reason: RouteCloseReason::Restart,
12424 ..
12425 }
12426 ));
12427 closing.flushed.take().unwrap().send(()).unwrap();
12428 };
12429 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
12430 result.unwrap();
12431 }
12432
12433 fn disconnect(&mut self) -> ClientControlPush {
12434 self.handler
12435 .cleanup_connection(self.module_ctx.connection_id)
12436 .unwrap();
12437 let closed = self.client_rx.try_recv().unwrap();
12438 assert_eq!(closed.header.ty, FrameType::Push);
12439 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
12440 assert!(matches!(
12441 &push,
12442 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
12443 if channels == &[self.route.client_channel]
12444 ));
12445 assert_eq!(
12446 self.client_rx.try_recv().unwrap().header.ty,
12447 FrameType::Goodbye
12448 );
12449 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
12450 push
12451 }
12452 }
12453
12454 #[cfg(unix)]
12455 #[tokio::test(start_paused = true)]
12456 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
12457 let mut fixture = route_closed_fixture().await;
12458 fixture.cut().await;
12459 assert!(matches!(
12460 fixture.disconnect(),
12461 ClientControlPush::RouteClosed {
12462 reason: RouteCloseReason::Restart,
12463 drained: true,
12464 ..
12465 }
12466 ));
12467 }
12468
12469 #[cfg(unix)]
12470 #[tokio::test(start_paused = true)]
12471 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
12472 let mut fixture = route_closed_fixture().await;
12473 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
12474 let started = tokio::time::Instant::now();
12475 fixture.cut().await;
12476 assert!(started.elapsed() >= Duration::from_secs(2));
12477 assert_eq!(fixture.route.flow.in_flight(), 1);
12478 assert!(matches!(
12479 fixture.disconnect(),
12480 ClientControlPush::RouteClosed {
12481 reason: RouteCloseReason::Restart,
12482 drained: false,
12483 ..
12484 }
12485 ));
12486 }
12487
12488 #[tokio::test]
12489 async fn module_connection_crash_reports_crash_and_not_drained() {
12490 let mut fixture = route_closed_fixture().await;
12491 assert!(matches!(
12495 fixture.disconnect(),
12496 ClientControlPush::RouteClosed {
12497 reason: RouteCloseReason::Crash,
12498 drained: false,
12499 ..
12500 }
12501 ));
12502 }
12503
12504 #[tokio::test]
12505 async fn dropping_router_connection_releases_registration() {
12506 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12507 let registry = Arc::new(Registry::default());
12508 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
12509 let router = Router::with_control_handler(Arc::clone(&control));
12510 let connection = router.begin_connection();
12511 let connection_id = connection.id();
12512 let (ctx, mut rx) = route_ctx(connection_id);
12513
12514 router
12515 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
12516 .await
12517 .unwrap();
12518 let response = rx.recv().await.unwrap();
12519 let ack = parse_ack(&response);
12520 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
12521 assert!(registry.get_module("aft").unwrap().is_some());
12522
12523 drop(connection);
12524
12525 assert!(registry.get_module("aft").unwrap().is_none());
12526 assert_eq!(registry.active_registration_count().unwrap(), 0);
12527
12528 control
12529 .cleanup_connection(ConnectionId::new(u64::MAX))
12530 .unwrap();
12531 let captured = crate::router::test_log::captured_logs(&logs);
12532 println!("captured registry lifecycle logs:\n{captured}");
12533 let events: Vec<_> = captured
12534 .lines()
12535 .filter(|line| {
12536 line.contains("module registered module_id=aft ")
12537 || line.contains("module registration ended")
12538 })
12539 .collect();
12540 assert_eq!(
12541 events.len(),
12542 2,
12543 "unexpected registry lifecycle logs: {captured}"
12544 );
12545 assert!(events[0].contains("module registered module_id=aft "));
12546 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
12547 assert!(events[1].contains(&format!(
12548 "module_id=aft connection_id={}",
12549 connection_id.get()
12550 )));
12551 assert!(events[1].contains("reason=connection_closed"));
12552 assert!(events[1].contains("module registration ended"));
12553 assert_eq!(
12554 captured
12555 .lines()
12556 .filter(|line| line.contains("module registered module_id=aft "))
12557 .count(),
12558 1,
12559 "legacy registration admission line must appear once: {captured}"
12560 );
12561 }
12562
12563 #[tokio::test]
12564 async fn hello_registration_keeps_legacy_module_registered_line_once() {
12565 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12566 let registry = Arc::new(Registry::default());
12567 let control = ControlHandler::new(Arc::clone(®istry));
12568 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
12569 hello_via_sink(
12570 &control,
12571 &ctx,
12572 &mut rx,
12573 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
12574 )
12575 .await;
12576
12577 let captured = crate::router::test_log::captured_logs(&logs);
12578 let admissions: Vec<_> = captured
12579 .lines()
12580 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
12581 .collect();
12582 assert_eq!(
12583 admissions.len(),
12584 1,
12585 "legacy admission line must remain exactly once: {captured}"
12586 );
12587 assert!(admissions[0].contains("routable_provider=true"));
12588 assert!(admissions[0].contains("connection_id=777"));
12589 }
12590
12591 fn capability_manifest(
12592 module_id: &str,
12593 provides: &[&str],
12594 must_never_reach: &[&str],
12595 ) -> ModuleManifest {
12596 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
12597 manifest.capabilities = Some(CapabilityDeclarations {
12598 provides: provides
12599 .iter()
12600 .map(|capability| (*capability).to_string())
12601 .collect(),
12602 requires: Vec::new(),
12603 must_never_reach: must_never_reach
12604 .iter()
12605 .map(|capability| (*capability).to_string())
12606 .collect(),
12607 });
12608 manifest
12609 }
12610
12611 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
12612 Frame::build(
12613 FrameType::Hello,
12614 control_flags(),
12615 0,
12616 0,
12617 corr,
12618 serde_json::to_vec(&ModuleHelloBody {
12619 protocol_ver: manifest.protocol_ver,
12620 manifest,
12621 control_ops: None,
12622 launch_nonce: None,
12623 })
12624 .expect("capability test HELLO serializes"),
12625 )
12626 .expect("capability test HELLO frame builds")
12627 }
12628
12629 fn catalog_update_with_capabilities_frame(
12630 corr: u64,
12631 capabilities: CapabilityDeclarations,
12632 ) -> Frame {
12633 Frame::build(
12634 FrameType::Request,
12635 control_flags(),
12636 0,
12637 0,
12638 corr,
12639 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
12640 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
12641 capabilities: Some(capabilities),
12642 ready: None,
12643 })
12644 .expect("capability catalog.update serializes"),
12645 )
12646 .expect("capability catalog.update frame builds")
12647 }
12648
12649 async fn register_capability_manifest(
12650 handler: &ControlHandler,
12651 ctx: &RouteCtx,
12652 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
12653 manifest: ModuleManifest,
12654 corr: u64,
12655 ) {
12656 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
12657 }
12658
12659 async fn open_route_for_capability_test(
12660 handler: &ControlHandler,
12661 target_ctx: &RouteCtx,
12662 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
12663 client_connection_id: u64,
12664 corr: u64,
12665 target_module_id: &str,
12666 consumer_identity: Option<ConsumerIdentity>,
12667 ) -> (
12668 mpsc::Receiver<crate::router::OutboundFrame>,
12669 ModuleControlRequest,
12670 ) {
12671 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
12672 let route_handler = handler.clone();
12673 let target_module_id = target_module_id.to_string();
12674 let route_task = tokio::spawn(async move {
12675 route_handler
12676 .handle_control_frame(
12677 &client_ctx,
12678 route_open_frame_with_admission_facts(
12679 corr,
12680 &target_module_id,
12681 unique_project_root("admission-facts"),
12682 consumer_identity,
12683 None,
12684 ),
12685 )
12686 .await
12687 .expect("capability test route.open succeeds")
12688 });
12689 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
12690 .await
12691 .expect("capability test route.open must reach route.bind")
12692 .expect("target control receiver stays open");
12693 let bind_request: ModuleControlRequest =
12694 serde_json::from_slice(&bind.body).expect("route.bind decodes");
12695 handler
12696 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
12697 .await
12698 .expect("capability test route.bind ACK succeeds");
12699 assert!(route_task.await.expect("route.open task joins").is_empty());
12700 let opened = client_rx
12701 .recv()
12702 .await
12703 .expect("successful route.open publishes a response");
12704 assert!(matches!(
12705 serde_json::from_slice::<ClientControlResponse>(&opened.body),
12706 Ok(ClientControlResponse::RouteOpen { .. })
12707 ));
12708 (client_rx, bind_request)
12709 }
12710
12711 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
12712 assert_eq!(frame.header.ty, FrameType::Push);
12713 assert_eq!(frame.header.channel, 0);
12714 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
12715 .expect("route.closed control push decodes");
12716 let ClientControlPush::RouteClosed { channels, .. } = &push else {
12717 panic!("expected route.closed");
12718 };
12719 assert_eq!(channels.len(), 1, "exactly one violating route closed");
12720 let channels = channels.clone();
12721 assert_eq!(
12722 push,
12723 ClientControlPush::RouteClosed {
12724 module_id: target_module_id.to_string(),
12725 channels,
12726 reason: RouteCloseReason::CapabilityDenied,
12727 drained: false,
12728 abandoned: 0,
12729 excluded_subscriptions: 0,
12730 terminal: Some(false),
12731 }
12732 );
12733 }
12734
12735 #[tokio::test]
12736 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
12737 let registry = Arc::new(Registry::default());
12738 let forwarding = Arc::new(ForwardingTable::default());
12739 let supervisor = SupervisorHandle::new();
12740 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12741 let handler =
12742 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12743 .with_supervisor(supervisor);
12744 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
12745 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
12746 register_capability_manifest(
12747 &handler,
12748 &target_ctx,
12749 &mut target_rx,
12750 capability_manifest("target", &["credentials-provider/v1"], &[]),
12751 1,
12752 )
12753 .await;
12754 register_capability_manifest(
12755 &handler,
12756 &opener_ctx,
12757 &mut opener_rx,
12758 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12759 2,
12760 )
12761 .await;
12762
12763 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
12764 let replies = handler
12765 .handle_control_frame(
12766 &client_ctx,
12767 route_open_frame_with_admission_facts(
12768 3,
12769 "target",
12770 unique_project_root("admission-facts"),
12771 Some(ConsumerIdentity {
12772 module_id: "opener".to_string(),
12773 launch_nonce: "opener-nonce".to_string(),
12774 }),
12775 None,
12776 ),
12777 )
12778 .await
12779 .expect("denied route.open returns a typed frame");
12780 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
12781 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12782 assert!(
12783 target_rx.try_recv().is_err(),
12784 "forbidden route.open must not relay route.bind"
12785 );
12786 }
12787
12788 #[tokio::test]
12789 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
12790 let registry = Arc::new(Registry::default());
12791 let forwarding = Arc::new(ForwardingTable::default());
12792 let supervisor = SupervisorHandle::new();
12793 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12794 let handler =
12795 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12796 .with_supervisor(supervisor);
12797 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
12798 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
12799 register_capability_manifest(
12800 &handler,
12801 &target_ctx,
12802 &mut target_rx,
12803 capability_manifest("target", &["credentials-provider/v1"], &[]),
12804 1,
12805 )
12806 .await;
12807 register_capability_manifest(
12808 &handler,
12809 &old_opener_ctx,
12810 &mut old_opener_rx,
12811 capability_manifest("opener", &[], &[]),
12812 2,
12813 )
12814 .await;
12815 let (mut client_rx, _) = open_route_for_capability_test(
12816 &handler,
12817 &target_ctx,
12818 &mut target_rx,
12819 712,
12820 3,
12821 "target",
12822 Some(ConsumerIdentity {
12823 module_id: "opener".to_string(),
12824 launch_nonce: "opener-nonce".to_string(),
12825 }),
12826 )
12827 .await;
12828 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12829
12830 handler
12831 .cleanup_connection(old_opener_ctx.connection_id)
12832 .expect("old opener registration cleans up");
12833 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
12834 register_capability_manifest(
12835 &handler,
12836 &new_opener_ctx,
12837 &mut new_opener_rx,
12838 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12839 4,
12840 )
12841 .await;
12842
12843 assert_capability_denied_push(
12844 client_rx
12845 .try_recv()
12846 .expect("HELLO deny addition must emit route.closed")
12847 .frame,
12848 "target",
12849 );
12850 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12851 assert!(matches!(
12852 target_rx.try_recv(),
12853 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
12854 ));
12855 }
12856
12857 #[tokio::test]
12858 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
12859 let registry = Arc::new(Registry::default());
12860 let forwarding = Arc::new(ForwardingTable::default());
12861 let supervisor = SupervisorHandle::new();
12862 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12863 let handler =
12864 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12865 .with_supervisor(supervisor);
12866 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
12867 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
12868 register_capability_manifest(
12869 &handler,
12870 &target_ctx,
12871 &mut target_rx,
12872 capability_manifest("target", &[], &[]),
12873 1,
12874 )
12875 .await;
12876 register_capability_manifest(
12877 &handler,
12878 &opener_ctx,
12879 &mut opener_rx,
12880 capability_manifest("opener", &[], &["credentials-provider/v1"]),
12881 2,
12882 )
12883 .await;
12884 let (mut client_rx, _) = open_route_for_capability_test(
12885 &handler,
12886 &target_ctx,
12887 &mut target_rx,
12888 722,
12889 3,
12890 "target",
12891 Some(ConsumerIdentity {
12892 module_id: "opener".to_string(),
12893 launch_nonce: "opener-nonce".to_string(),
12894 }),
12895 )
12896 .await;
12897 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12898
12899 let replies = handler
12900 .handle_control_frame(
12901 &target_ctx,
12902 catalog_update_with_capabilities_frame(
12903 4,
12904 CapabilityDeclarations {
12905 provides: vec!["credentials-provider/v1".to_string()],
12906 requires: Vec::new(),
12907 must_never_reach: Vec::new(),
12908 },
12909 ),
12910 )
12911 .await
12912 .expect("claim catalog.update succeeds");
12913 assert!(matches!(
12914 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
12915 Ok(ModuleControlResponseToModule::CatalogUpdate {})
12916 ));
12917 assert_capability_denied_push(
12918 client_rx
12919 .try_recv()
12920 .expect("claim addition must emit route.closed")
12921 .frame,
12922 "target",
12923 );
12924 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
12925 assert!(matches!(
12926 target_rx.try_recv(),
12927 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
12928 ));
12929 }
12930
12931 #[tokio::test]
12932 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
12933 let registry = Arc::new(Registry::default());
12934 let forwarding = Arc::new(ForwardingTable::default());
12935 let supervisor = SupervisorHandle::new();
12936 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
12937 let handler =
12938 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12939 .with_supervisor(supervisor);
12940 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
12941 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
12942 register_capability_manifest(
12943 &handler,
12944 &target_ctx,
12945 &mut target_rx,
12946 capability_manifest("target", &["credentials-provider/v1"], &[]),
12947 1,
12948 )
12949 .await;
12950 register_capability_manifest(
12951 &handler,
12952 &opener_ctx,
12953 &mut opener_rx,
12954 capability_manifest("opener", &[], &[]),
12955 2,
12956 )
12957 .await;
12958 let (mut client_rx, _) = open_route_for_capability_test(
12959 &handler,
12960 &target_ctx,
12961 &mut target_rx,
12962 732,
12963 3,
12964 "target",
12965 Some(ConsumerIdentity {
12966 module_id: "opener".to_string(),
12967 launch_nonce: "opener-nonce".to_string(),
12968 }),
12969 )
12970 .await;
12971 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
12972
12973 handler
12974 .handle_control_frame(
12975 &target_ctx,
12976 catalog_update_with_capabilities_frame(
12977 4,
12978 CapabilityDeclarations {
12979 provides: Vec::new(),
12980 requires: Vec::new(),
12981 must_never_reach: Vec::new(),
12982 },
12983 ),
12984 )
12985 .await
12986 .expect("claim removal catalog.update succeeds");
12987 assert_eq!(
12988 forwarding.active_binding_count().unwrap(),
12989 1,
12990 "removing an attested target claim must leave the route census unchanged"
12991 );
12992 assert!(
12993 client_rx.try_recv().is_err(),
12994 "claim removal must not emit route.closed capability_denied"
12995 );
12996 assert!(
12997 target_rx.try_recv().is_err(),
12998 "claim removal must not send the target a route GOODBYE"
12999 );
13000 }
13001
13002 #[tokio::test]
13005 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
13006 let registry = Arc::new(Registry::default());
13007 let forwarding = Arc::new(ForwardingTable::default());
13008 let supervisor = SupervisorHandle::new();
13009 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13010 let handler =
13011 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13012 .with_supervisor(supervisor);
13013 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
13014 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
13015 register_capability_manifest(
13016 &handler,
13017 &target_ctx,
13018 &mut target_rx,
13019 capability_manifest("target", &["credentials-provider/v1"], &[]),
13020 1,
13021 )
13022 .await;
13023 register_capability_manifest(
13024 &handler,
13025 &opener_ctx,
13026 &mut opener_rx,
13027 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13028 2,
13029 )
13030 .await;
13031
13032 let (_client_rx, bind) = open_route_for_capability_test(
13033 &handler,
13034 &target_ctx,
13035 &mut target_rx,
13036 742,
13037 3,
13038 "target",
13039 None,
13040 )
13041 .await;
13042 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
13043 panic!("direct scope-honesty route must bind");
13044 };
13045 assert_eq!(principal, Some(Principal::Direct));
13046 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13047 }
13048
13049 #[tokio::test]
13052 async fn must_never_reach_self_route_is_capability_forbidden() {
13053 let registry = Arc::new(Registry::default());
13054 let forwarding = Arc::new(ForwardingTable::default());
13055 let supervisor = SupervisorHandle::new();
13056 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
13057 let handler =
13058 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13059 .with_supervisor(supervisor);
13060 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
13061 register_capability_manifest(
13062 &handler,
13063 &self_ctx,
13064 &mut self_rx,
13065 capability_manifest(
13066 "self-provider",
13067 &["credentials-provider/v1"],
13068 &["credentials-provider/v1"],
13069 ),
13070 1,
13071 )
13072 .await;
13073
13074 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
13075 let replies = handler
13076 .handle_control_frame(
13077 &client_ctx,
13078 route_open_frame_with_admission_facts(
13079 2,
13080 "self-provider",
13081 unique_project_root("admission-facts"),
13082 Some(ConsumerIdentity {
13083 module_id: "self-provider".to_string(),
13084 launch_nonce: "self-nonce".to_string(),
13085 }),
13086 None,
13087 ),
13088 )
13089 .await
13090 .expect("self-route refusal returns a typed frame");
13091 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13092 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13093 assert!(
13094 self_rx.try_recv().is_err(),
13095 "self denial must not relay route.bind"
13096 );
13097 }
13098
13099 #[test]
13100 fn unsupported_channel_zero_frame_returns_error() {
13101 let handler = ControlHandler::default();
13102 let request = Frame::build(
13103 FrameType::Request,
13104 control_flags(),
13105 0,
13106 0,
13107 21,
13108 b"opaque".to_vec(),
13109 )
13110 .unwrap();
13111
13112 let response = handler
13113 .handle_control(ConnectionId::new(1), request)
13114 .unwrap();
13115
13116 assert_eq!(response[0].header.ty, FrameType::Error);
13117 assert_eq!(
13118 parse_error(&response[0])["code"],
13119 "unsupported_control_frame"
13120 );
13121 }
13122
13123 mod swap {
13128 use super::*;
13129
13130 const INCUMBENT: ConnectionId = ConnectionId::new(30);
13131 const CANDIDATE: ConnectionId = ConnectionId::new(40);
13132
13133 struct Swap {
13134 registry: Arc<Registry>,
13135 forwarding: Arc<ForwardingTable>,
13136 handler: ControlHandler,
13137 incumbent_ctx: RouteCtx,
13138 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13139 candidate_ctx: RouteCtx,
13140 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13141 }
13142
13143 async fn swap_with_incumbent() -> Swap {
13144 let registry = Arc::new(Registry::default());
13145 let forwarding = Arc::new(ForwardingTable::default());
13146 let handler =
13147 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13148 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
13149 hello_via_sink(
13150 &handler,
13151 &incumbent_ctx,
13152 &mut incumbent_rx,
13153 hello_frame("aft", PROTOCOL_VERSION, 7),
13154 )
13155 .await;
13156 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
13157 Swap {
13158 registry,
13159 forwarding,
13160 handler,
13161 incumbent_ctx,
13162 incumbent_rx,
13163 candidate_ctx,
13164 candidate_rx,
13165 }
13166 }
13167
13168 fn register_candidate(swap: &Swap, ready: Option<bool>) {
13169 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
13170 candidate_manifest.ready = ready;
13171 let registration = swap
13172 .registry
13173 .register_candidate_with_control_ops(
13174 candidate_manifest,
13175 PROTOCOL_VERSION,
13176 CANDIDATE,
13177 module_baseline_control_ops(),
13178 )
13179 .unwrap();
13180 swap.forwarding
13181 .register_candidate_module_connection(
13182 CANDIDATE,
13183 "aft".to_string(),
13184 PROTOCOL_VERSION,
13185 manifest_concurrency(®istration.manifest),
13186 swap.candidate_ctx.egress.clone(),
13187 )
13188 .unwrap();
13189 }
13190
13191 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
13192 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
13193 swap.registry.promote_candidate("aft").unwrap().unwrap();
13194 cutover.incumbent.unwrap()
13195 }
13196
13197 fn keyed_total(counters: &Value, key: &str) -> u64 {
13198 counters[key]
13199 .as_object()
13200 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
13201 .unwrap_or(0)
13202 }
13203
13204 #[tokio::test]
13205 async fn replacement_logs_old_end_and_new_admission_once() {
13206 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13207 let mut swap = swap_with_incumbent().await;
13208 swap.handler
13209 .supervisor
13210 .open_swap("aft", "candidate-nonce".to_string());
13211 hello_via_sink(
13212 &swap.handler,
13213 &swap.candidate_ctx,
13214 &mut swap.candidate_rx,
13215 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
13216 )
13217 .await;
13218 cutover(&swap);
13219 swap.handler.cleanup_connection(INCUMBENT).unwrap();
13220
13221 let captured = crate::router::test_log::captured_logs(&logs);
13222 println!("captured replacement registry logs:\n{captured}");
13223 let new_admissions: Vec<_> = captured
13224 .lines()
13225 .filter(|line| {
13226 line.contains("connection_id=40")
13227 && line.contains("swap candidate registered; not routable until cutover")
13228 })
13229 .collect();
13230 assert_eq!(
13231 new_admissions.len(),
13232 1,
13233 "unexpected admission logs: {captured}"
13234 );
13235 assert!(new_admissions[0].contains("module_id=aft"));
13236 assert!(new_admissions[0].contains("connection_id=40"));
13237 assert!(
13238 new_admissions[0].contains("swap candidate registered; not routable until cutover")
13239 );
13240 assert!(new_admissions[0].contains("ready=true"));
13241
13242 let old_admissions: Vec<_> = captured
13243 .lines()
13244 .filter(|line| {
13245 line.contains("module registered module_id=aft ")
13246 && line.contains("connection_id=30")
13247 })
13248 .collect();
13249 assert_eq!(
13250 old_admissions.len(),
13251 1,
13252 "unexpected incumbent admission: {captured}"
13253 );
13254
13255 let old_ends: Vec<_> = captured
13256 .lines()
13257 .filter(|line| {
13258 line.contains("connection_id=30") && line.contains("module registration ended")
13259 })
13260 .collect();
13261 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
13262 assert!(old_ends[0].contains("module_id=aft"));
13263 assert!(old_ends[0].contains("connection_id=30"));
13264 assert!(old_ends[0].contains("reason=replaced"));
13265 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
13266 assert_eq!(
13267 captured
13268 .lines()
13269 .filter(|line| line.contains("module registration promoted"))
13270 .count(),
13271 1,
13272 "unexpected promotion logs: {captured}"
13273 );
13274 }
13275
13276 #[tokio::test]
13282 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
13283 let mut swap = swap_with_incumbent().await;
13284 let handler = swap.handler.clone();
13285
13286 let cotenant = ConnectionId::new(31);
13288 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
13289 let (cotenant_task, cotenant_bind) = relay_route_open(
13290 &handler,
13291 cotenant,
13292 &cotenant_ctx.egress,
13293 &mut swap.incumbent_rx,
13294 100,
13295 "aft",
13296 "swap-cotenant",
13297 )
13298 .await;
13299 handler
13300 .handle_control_frame(
13301 &swap.incumbent_ctx,
13302 route_bind_ack(cotenant_bind.header.corr),
13303 )
13304 .await
13305 .unwrap();
13306 assert!(cotenant_task.await.unwrap().is_empty());
13307 let (cotenant_channel, cotenant_epoch) =
13308 published_route(&cotenant_rx.recv().await.unwrap());
13309
13310 let caller = ConnectionId::new(32);
13312 let (caller_ctx, mut caller_rx) = route_ctx(caller);
13313 let (caller_task, caller_bind) = relay_route_open(
13314 &handler,
13315 caller,
13316 &caller_ctx.egress,
13317 &mut swap.incumbent_rx,
13318 101,
13319 "aft",
13320 "swap-caller",
13321 )
13322 .await;
13323 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
13324
13325 register_candidate(&swap, None);
13326 cutover(&swap);
13327
13328 let ack = handler
13330 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
13331 .await;
13332 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
13333 if module_loop_error.is_some() {
13334 handler.cleanup_connection(INCUMBENT).unwrap();
13337 }
13338
13339 assert!(
13341 cotenant_rx.try_recv().is_err(),
13342 "the co-tenant route on the incumbent was torn down by one late ack: \
13343 {module_loop_error:?}"
13344 );
13345 assert!(matches!(
13346 swap.forwarding
13347 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
13348 .unwrap(),
13349 DataRoute::Client(DataRouteState::Bound(_))
13350 ));
13351 assert_eq!(module_loop_error, None);
13352 assert!(swap
13353 .registry
13354 .get_module_by_connection(INCUMBENT)
13355 .unwrap()
13356 .is_some());
13357
13358 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
13360 .await
13361 .expect("the incumbent is told to drop the abandoned binding")
13362 .unwrap()
13363 .frame;
13364 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13365 assert_eq!(goodbye.header.channel, abandoned_channel);
13366 assert_eq!(goodbye.header.epoch, abandoned_epoch);
13367 assert!(swap.incumbent_rx.try_recv().is_err());
13368
13369 let response = caller_task.await.unwrap();
13371 assert_eq!(response.len(), 1);
13372 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
13373 assert!(caller_rx.try_recv().is_err());
13374
13375 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
13379 let counters = handler.counters().snapshot();
13380 assert_eq!(
13381 keyed_total(&counters, "route_open_accepted_by_principal"),
13382 1
13383 );
13384 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
13385 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
13386 }
13387
13388 #[tokio::test]
13392 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
13393 let mut swap = swap_with_incumbent().await;
13394 register_candidate(&swap, None);
13395 let incumbent = cutover(&swap);
13396 swap.forwarding
13397 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
13398 .unwrap()
13399 .expect("the incumbent is still registered");
13400
13401 let client = ConnectionId::new(33);
13402 let (client_ctx, mut client_rx) = route_ctx(client);
13403 let route_handler = swap.handler.clone();
13404 let open_ctx = RouteCtx {
13405 connection_id: client,
13406 egress: client_ctx.egress.clone(),
13407 };
13408 let mut route_task = tokio::spawn(async move {
13409 route_handler
13410 .handle_control_frame(
13411 &open_ctx,
13412 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
13413 )
13414 .await
13415 .unwrap()
13416 });
13417 let bind = tokio::select! {
13418 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
13419 response = &mut route_task => {
13420 let response = response.unwrap();
13421 panic!(
13422 "post-cutover route.open was refused instead of relayed to the candidate: {}",
13423 parse_error(&response[0])["code"]
13424 );
13425 }
13426 };
13427 swap.handler
13428 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
13429 .await
13430 .unwrap();
13431 assert!(route_task.await.unwrap().is_empty());
13432 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
13433 match swap
13434 .forwarding
13435 .lookup_data_route(client, channel, epoch)
13436 .unwrap()
13437 {
13438 DataRoute::Client(DataRouteState::Bound(route)) => {
13439 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
13440 }
13441 other => panic!("expected a bound route on the candidate, got {other:?}"),
13442 }
13443 assert!(swap.incumbent_rx.try_recv().is_err());
13444 }
13445
13446 #[tokio::test]
13451 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
13452 let swap = swap_with_incumbent().await;
13453 register_candidate(&swap, Some(false));
13454 let update = Frame::build(
13455 FrameType::Request,
13456 control_flags(),
13457 0,
13458 0,
13459 55,
13460 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13461 provides: manifest("aft", PROTOCOL_VERSION).provides,
13462 capabilities: None,
13463 ready: Some(true),
13464 })
13465 .unwrap(),
13466 )
13467 .unwrap();
13468
13469 let replies = swap
13470 .handler
13471 .handle_control_frame(&swap.candidate_ctx, update)
13472 .await
13473 .unwrap();
13474
13475 assert_eq!(replies.len(), 1);
13476 assert_eq!(
13477 replies[0].header.ty,
13478 FrameType::Response,
13479 "candidate catalog.update was refused: {:?}",
13480 serde_json::from_slice::<Value>(&replies[0].body).ok()
13481 );
13482 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
13483 assert_eq!(
13484 swap.registry
13485 .get_module("aft")
13486 .unwrap()
13487 .unwrap()
13488 .connection_id,
13489 INCUMBENT
13490 );
13491 }
13492 }
13493
13494 mod swap_admission {
13498 use super::*;
13499
13500 const INCUMBENT_NONCE: &str = "incumbent-nonce";
13501 const CANDIDATE_NONCE: &str = "candidate-nonce";
13502
13503 fn handler_with_incumbent(
13504 module_id: &str,
13505 reserved: bool,
13506 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
13507 let registry = Arc::new(Registry::default());
13508 let supervisor = SupervisorHandle::new();
13509 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
13510 if reserved {
13511 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
13512 }
13513 let handler =
13514 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
13515 let incumbent = handler
13516 .handle_control(
13517 ConnectionId::new(1),
13518 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
13519 )
13520 .unwrap();
13521 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
13522 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
13523 (registry, supervisor, handler)
13524 }
13525
13526 #[test]
13532 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
13533 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
13534
13535 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
13536 let replies = handler
13537 .handle_control(
13538 ConnectionId::new(connection),
13539 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
13540 )
13541 .unwrap();
13542 assert_eq!(replies[0].header.ty, FrameType::Error);
13543 assert_eq!(
13544 parse_error(&replies[0])["code"],
13545 "swap_token_invalid",
13546 "nonce {nonce:?}"
13547 );
13548 }
13549 assert!(registry.get_candidate("aft").unwrap().is_none());
13550 assert_eq!(
13551 registry.get_module("aft").unwrap().unwrap().connection_id,
13552 ConnectionId::new(1)
13553 );
13554
13555 let admitted = handler
13558 .handle_control(
13559 ConnectionId::new(4),
13560 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
13561 )
13562 .unwrap();
13563 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
13564 assert_eq!(
13565 registry
13566 .get_candidate("aft")
13567 .unwrap()
13568 .unwrap()
13569 .connection_id,
13570 ConnectionId::new(4)
13571 );
13572 assert_eq!(
13573 registry.get_module("aft").unwrap().unwrap().connection_id,
13574 ConnectionId::new(1),
13575 "the candidate must not take the active slot"
13576 );
13577 let replayed = handler
13578 .handle_control(
13579 ConnectionId::new(5),
13580 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
13581 )
13582 .unwrap();
13583 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
13584
13585 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
13589 let squatter = handler
13590 .handle_control(
13591 ConnectionId::new(6),
13592 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
13593 )
13594 .unwrap();
13595 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
13596 assert!(
13597 registry.get_module("aft").unwrap().is_none(),
13598 "a squatter took the active slot of an id being swapped"
13599 );
13600 }
13601
13602 #[test]
13607 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
13608 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
13609
13610 let replies = handler
13611 .handle_control(
13612 ConnectionId::new(2),
13613 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
13614 )
13615 .unwrap();
13616
13617 assert_eq!(
13618 replies[0].header.ty,
13619 FrameType::HelloAck,
13620 "reserved candidate refused: {:?}",
13621 serde_json::from_slice::<Value>(&replies[0].body).ok()
13622 );
13623 assert_eq!(
13624 registry
13625 .get_candidate("vault")
13626 .unwrap()
13627 .unwrap()
13628 .connection_id,
13629 ConnectionId::new(2)
13630 );
13631 }
13632
13633 #[test]
13636 fn without_an_open_swap_the_ordinary_gates_decide() {
13637 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
13638 supervisor.close_swap("vault");
13639
13640 let candidate = handler
13641 .handle_control(
13642 ConnectionId::new(2),
13643 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
13644 )
13645 .unwrap();
13646 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
13647 let duplicate = handler
13648 .handle_control(
13649 ConnectionId::new(3),
13650 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
13651 )
13652 .unwrap();
13653 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13654 assert!(registry.get_candidate("vault").unwrap().is_none());
13655 }
13656 }
13657
13658 mod scopes {
13662 mod activity {
13663 include!("activity_scope_tests.rs");
13664 }
13665 mod delta {
13666 include!("scope_handler_delta_tests.rs");
13667 }
13668 mod call_modes {
13669 include!("scope_call_mode_tests.rs");
13670 }
13671
13672 use subc_protocol::scope::{
13673 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
13674 ScopeStatus,
13675 };
13676
13677 use super::*;
13678
13679 const OWNER: &str = "prefrontal-core";
13680
13681 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
13682 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
13683 }
13684
13685 async fn call(
13686 handler: &ControlHandler,
13687 ctx: &RouteCtx,
13688 request: &ModuleControlRequestFromModule,
13689 ) -> Frame {
13690 let body = serde_json::to_vec(request).unwrap();
13691 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
13692 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
13693 assert_eq!(replies.len(), 1, "{replies:?}");
13694 replies.pop().unwrap()
13695 }
13696
13697 async fn sync(
13698 handler: &ControlHandler,
13699 ctx: &RouteCtx,
13700 generation: u64,
13701 scopes: Vec<ScopeRecord>,
13702 ) -> Result<ModuleControlResponseToModule, String> {
13703 let reply = call(
13704 handler,
13705 ctx,
13706 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
13707 )
13708 .await;
13709 match reply.header.ty {
13710 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
13711 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
13712 }
13713 }
13714
13715 async fn describe(
13716 handler: &ControlHandler,
13717 ctx: &RouteCtx,
13718 owner: &str,
13719 scope_ref: &str,
13720 ) -> ModuleControlResponseToModule {
13721 let reply = call(
13722 handler,
13723 ctx,
13724 &ModuleControlRequestFromModule::ScopeDescribe {
13725 owner: Principal::Reserved {
13726 module_id: owner.to_string(),
13727 },
13728 scope_ref: scope_ref.to_string(),
13729 },
13730 )
13731 .await;
13732 assert_eq!(
13733 reply.header.ty,
13734 FrameType::Response,
13735 "{:?}",
13736 parse_error(&reply)
13737 );
13738 serde_json::from_slice(&reply.body).unwrap()
13739 }
13740
13741 async fn module(
13743 handler: &ControlHandler,
13744 connection: u64,
13745 module_id: &str,
13746 nonce: Option<&str>,
13747 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
13748 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
13749 hello_via_sink(
13750 handler,
13751 &ctx,
13752 &mut rx,
13753 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
13754 )
13755 .await;
13756 (ctx, rx)
13757 }
13758
13759 #[tokio::test]
13762 async fn a_client_connection_cannot_sync_or_describe() {
13763 let handler = ControlHandler::new(Arc::new(Registry::default()));
13764 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
13765 for request in [
13766 ModuleControlRequestFromModule::ScopeSync {
13767 generation: 1,
13768 scopes: vec![head("s", 1)],
13769 },
13770 ModuleControlRequestFromModule::ScopeDescribe {
13771 owner: Principal::Direct,
13772 scope_ref: "s".to_string(),
13773 },
13774 ] {
13775 let reply = call(&handler, &ctx, &request).await;
13776 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
13777 }
13778 assert!(
13779 !handler
13780 .scopes
13781 .read()
13782 .unwrap()
13783 .describe(
13784 &Principal::Reserved {
13785 module_id: OWNER.to_string()
13786 },
13787 "s"
13788 )
13789 .owner_synced
13790 );
13791 }
13792
13793 #[tokio::test]
13796 async fn a_module_without_a_supervised_launch_cannot_sync() {
13797 let handler = ControlHandler::new(Arc::new(Registry::default()));
13798 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
13799 assert_eq!(
13800 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
13801 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13802 );
13803 }
13804
13805 #[tokio::test]
13806 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
13807 let supervisor = SupervisorHandle::new();
13808 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13809 let handler = ControlHandler::new(Arc::new(Registry::default()))
13810 .with_supervisor(supervisor.clone());
13811 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
13812 sync(&handler, &incumbent, 1, vec![head("s", 1)])
13813 .await
13814 .expect("the current launch syncs");
13815
13816 supervisor.open_swap(OWNER, "n2".to_string());
13819 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
13820 assert_eq!(
13821 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
13822 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13823 );
13824 sync(&handler, &incumbent, 2, vec![head("s", 1)])
13825 .await
13826 .expect("the serving owner syncs during the swap");
13827
13828 supervisor.close_swap(OWNER);
13831 assert_eq!(
13832 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
13833 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13834 );
13835 sync(&handler, &incumbent, 3, vec![head("s", 1)])
13836 .await
13837 .expect("the serving owner syncs after the rollback");
13838 handler.cleanup_connection(candidate.connection_id).unwrap();
13839
13840 supervisor.open_swap(OWNER, "n3".to_string());
13845 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
13846 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
13847 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
13848 .await
13849 .expect("the promoted launch takes authority");
13850 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
13851 panic!("unexpected reply {reply:?}");
13852 };
13853 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
13854 assert_eq!(
13855 sync(&handler, &incumbent, 4, Vec::new()).await,
13856 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
13857 );
13858 }
13859
13860 #[tokio::test]
13863 async fn closing_the_authority_connection_frees_sync_authority() {
13864 let supervisor = SupervisorHandle::new();
13865 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13866 let handler = ControlHandler::new(Arc::new(Registry::default()))
13867 .with_supervisor(supervisor.clone());
13868 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
13869 sync(&handler, &first, 10, vec![head("s", 1)])
13870 .await
13871 .unwrap();
13872 handler.cleanup_connection(first.connection_id).unwrap();
13873
13874 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
13875 sync(&handler, &second, 1, vec![head("s", 1)])
13876 .await
13877 .expect("the next connection takes the released authority");
13878 }
13879
13880 #[tokio::test]
13881 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
13882 let supervisor = SupervisorHandle::new();
13883 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
13884 let handler =
13885 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
13886 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
13887 sync(&handler, &first, 10, vec![head("s", 1)])
13888 .await
13889 .unwrap();
13890 handler
13891 .handle_control_frame(
13892 &first,
13893 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
13894 )
13895 .await
13896 .unwrap();
13897 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
13898 sync(&handler, &second, 1, vec![head("s", 1)])
13899 .await
13900 .expect("GOODBYE releases authority even if the old socket remains open");
13901 }
13902
13903 #[tokio::test]
13904 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
13905 let registry = Arc::new(Registry::default());
13906 let supervisor_handle = SupervisorHandle::new();
13907 let supervisor =
13908 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
13909 .with_handle(supervisor_handle.clone())
13910 .with_daemon_incarnation("incarnation-7".to_string());
13911 supervisor
13914 .supervise_configured(
13915 ModuleSpec {
13916 module_id: OWNER.to_string(),
13917 program: PathBuf::from("/nonexistent/prefrontal-core"),
13918 args: Vec::new(),
13919 env: Vec::new(),
13920 reserved: false,
13921 reserved_prefixes: Vec::new(),
13922 protocol: ModuleProtocol::Subc,
13923 overlap: Default::default(),
13924 },
13925 false,
13926 )
13927 .unwrap();
13928 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
13929 let handler =
13930 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13931 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
13932
13933 let ModuleControlResponseToModule::ScopeDescribe {
13935 status,
13936 daemon_incarnation,
13937 owner_synced,
13938 owner_configured,
13939 scope,
13940 ..
13941 } = describe(&handler, &reader, OWNER, "s").await
13942 else {
13943 panic!("not a describe reply");
13944 };
13945 assert_eq!(status, ScopeStatus::NotLive);
13946 assert_eq!(daemon_incarnation, "incarnation-7");
13947 assert!(!owner_synced);
13948 assert!(owner_configured);
13949 assert!(scope.is_none());
13950
13951 let ModuleControlResponseToModule::ScopeDescribe {
13954 status,
13955 owner_configured,
13956 ..
13957 } = describe(&handler, &reader, "ghost", "s").await
13958 else {
13959 panic!("not a describe reply");
13960 };
13961 assert_eq!(status, ScopeStatus::NotLive);
13962 assert!(!owner_configured);
13963
13964 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
13966 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
13967 let ModuleControlResponseToModule::ScopeDescribe {
13968 status,
13969 scope_epoch,
13970 owner_synced,
13971 scope,
13972 ..
13973 } = describe(&handler, &reader, OWNER, "s").await
13974 else {
13975 panic!("not a describe reply");
13976 };
13977 assert_eq!(status, ScopeStatus::Live);
13978 assert_eq!(scope_epoch, Some(4));
13979 assert!(owner_synced);
13980 let stamp = scope.expect("a live scope carries its stamp");
13981 assert!(
13982 stamp.owner_authorized,
13983 "prefrontal-core is the default authority"
13984 );
13985 assert_eq!(stamp.kind, ScopeKind::Head);
13986 }
13987
13988 #[tokio::test]
13989 async fn scope_authority_owners_decides_owner_authorized() {
13990 let supervisor = SupervisorHandle::new();
13991 supervisor.set_spawn_nonce("broca", "b1".to_string());
13992 let handler = ControlHandler::new(Arc::new(Registry::default()))
13993 .with_supervisor(supervisor)
13994 .with_scope_authority_owners(vec!["broca".to_string()]);
13995 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
13996 let mut gated = head("s", 1);
13997 gated.attributes.agent_id = Some("agent".to_string());
13998 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
13999 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
14000 describe(&handler, &broca, "broca", "s").await
14001 else {
14002 panic!("not a describe reply");
14003 };
14004 assert!(scope.unwrap().owner_authorized);
14005 }
14006
14007 #[tokio::test]
14011 async fn scope_ops_and_the_scopes_capability_are_advertised() {
14012 let handler = ControlHandler::new(Arc::new(Registry::default()));
14013 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
14014 let ack = hello_via_sink(
14015 &handler,
14016 &ctx,
14017 &mut rx,
14018 hello_frame("m", PROTOCOL_VERSION, 1),
14019 )
14020 .await;
14021 let ack = parse_ack(&ack);
14022 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
14023 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
14024 }
14025 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
14026
14027 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
14028 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
14029 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
14030 let reply = handler
14031 .handle_control_frame(&client, frame)
14032 .await
14033 .unwrap()
14034 .pop()
14035 .unwrap();
14036 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14037 serde_json::from_slice(&reply.body).unwrap()
14038 else {
14039 panic!("not a server.describe reply");
14040 };
14041 assert!(
14042 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
14043 "{capabilities:?}"
14044 );
14045 }
14046
14047 const PLEXUS: &str = "plexus";
14050 const OTHER: &str = "other";
14051 const AFT: &str = "aft";
14052 const BROCA: &str = "broca";
14053 const MAGIC: &str = "magic-context";
14054
14055 fn nonce(module_id: &str) -> String {
14056 format!("nonce-{module_id}")
14057 }
14058
14059 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
14060 let (tx, rx) = mpsc::channel(64);
14061 (
14062 RouteCtx {
14063 connection_id: ConnectionId::new(connection),
14064 egress: FrameSink::new(tx),
14065 },
14066 rx,
14067 )
14068 }
14069
14070 struct Rig {
14074 handler: ControlHandler,
14075 forwarding: Arc<ForwardingTable>,
14076 owner: RouteCtx,
14077 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14078 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
14079 generation: u64,
14080 next_connection: u64,
14081 _supervisor: Supervisor,
14082 }
14083
14084 async fn rig() -> Rig {
14085 rig_with_flow_support(true).await
14086 }
14087
14088 async fn rig_with_flow_support(flow_support: bool) -> Rig {
14089 rig_with_scope_capabilities(flow_support, false).await
14090 }
14091
14092 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
14093 let mut provides = Vec::new();
14094 if flow_support {
14095 provides.push("flow-scopes/v1");
14096 }
14097 if run_support {
14098 provides.push("agent-run-scopes/v1");
14099 }
14100 rig_with_target_capabilities(&provides).await
14101 }
14102
14103 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
14104 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
14105 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
14106 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
14109 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
14110 Frame::build(
14111 FrameType::Hello,
14112 control_flags(),
14113 0,
14114 0,
14115 connection,
14116 serde_json::to_vec(&body).unwrap(),
14117 )
14118 .unwrap()
14119 }
14120
14121 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
14122 let registry = Arc::new(Registry::default());
14123 let forwarding = Arc::new(ForwardingTable::default());
14124 let supervisor_handle = SupervisorHandle::new();
14125 let supervisor =
14126 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14127 .with_handle(supervisor_handle.clone());
14128 supervisor
14129 .supervise_configured(
14130 ModuleSpec {
14131 module_id: OWNER.to_string(),
14132 program: PathBuf::from("/nonexistent/prefrontal-core"),
14133 args: Vec::new(),
14134 env: Vec::new(),
14135 reserved: false,
14136 reserved_prefixes: Vec::new(),
14137 protocol: ModuleProtocol::Subc,
14138 overlap: Default::default(),
14139 },
14140 false,
14141 )
14142 .unwrap();
14143 for module_id in [OWNER, AFT, BROCA, MAGIC] {
14144 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
14145 }
14146 let handler =
14147 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14148 .with_supervisor(supervisor_handle);
14149 let (owner, mut owner_rx) = wide_ctx(1);
14150 hello_via_sink(
14151 &handler,
14152 &owner,
14153 &mut owner_rx,
14154 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
14155 )
14156 .await;
14157 let mut modules = BTreeMap::new();
14158 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
14159 let (ctx, mut rx) = wide_ctx(connection);
14160 let hello = hello_with_capabilities(module_id, connection, provides);
14161 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
14162 modules.insert(module_id.to_string(), (ctx, rx));
14163 }
14164 Rig {
14165 handler,
14166 forwarding,
14167 owner,
14168 _owner_rx: owner_rx,
14169 modules,
14170 generation: 0,
14171 next_connection: 100,
14172 _supervisor: supervisor,
14173 }
14174 }
14175
14176 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
14177 ScopeCarrier::new(Principal::Reserved {
14178 module_id: module_id.to_string(),
14179 })
14180 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
14181 }
14182
14183 fn session(scope_epoch: u64) -> ScopeRecord {
14186 let mut record = head("s", scope_epoch);
14187 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
14188 record.attributes.agent_id = Some("agent-1".to_string());
14189 record.attributes.delegates = true;
14190 record
14191 }
14192
14193 impl Rig {
14194 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
14195 self.generation += 1;
14196 sync(&self.handler, &self.owner, self.generation, scopes)
14197 .await
14198 .expect("the owner's sync is accepted");
14199 }
14200
14201 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
14202 ScopeSelector {
14203 owner: Principal::Reserved {
14204 module_id: OWNER.to_string(),
14205 },
14206 scope_ref: scope_ref.to_string(),
14207 scope_epoch,
14208 }
14209 }
14210
14211 fn open_frame(
14212 &mut self,
14213 opener: Option<&str>,
14214 target: &str,
14215 scope: Option<ScopeSelector>,
14216 ) -> (
14217 RouteCtx,
14218 mpsc::Receiver<crate::router::OutboundFrame>,
14219 Frame,
14220 ) {
14221 self.next_connection += 1;
14222 let (ctx, rx) = wide_ctx(self.next_connection);
14223 let root = unique_project_root("scoped-open");
14224 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
14225 target: RouteTarget::ToolProvider {
14226 module_id: target.to_string(),
14227 },
14228 identity: BindIdentity::new(
14229 root.path().to_path_buf(),
14230 "unit".to_string(),
14231 "session".to_string(),
14232 ),
14233 consumer_identity: opener.map(|module_id| ConsumerIdentity {
14234 module_id: module_id.to_string(),
14235 launch_nonce: nonce(module_id),
14236 }),
14237 consumer_capabilities: None,
14238 role_versions: None,
14239 admission_facts: None,
14240 scope,
14241 })
14242 .unwrap();
14243 let frame = Frame::build(
14244 FrameType::Request,
14245 control_flags(),
14246 0,
14247 0,
14248 self.next_connection,
14249 body,
14250 )
14251 .unwrap();
14252 (ctx, rx, frame)
14253 }
14254
14255 async fn refused(
14257 &mut self,
14258 opener: Option<&str>,
14259 target: &str,
14260 scope: Option<ScopeSelector>,
14261 ) -> String {
14262 self.refusal_body(opener, target, scope).await["code"]
14263 .as_str()
14264 .unwrap()
14265 .to_string()
14266 }
14267
14268 async fn refusal_body(
14269 &mut self,
14270 opener: Option<&str>,
14271 target: &str,
14272 scope: Option<ScopeSelector>,
14273 ) -> Value {
14274 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
14275 let replies = tokio::time::timeout(
14276 Duration::from_secs(2),
14277 self.handler.handle_control_frame(&ctx, frame),
14278 )
14279 .await
14280 .expect("the open must be refused before waiting for a bind ack")
14281 .unwrap();
14282 assert_eq!(replies.len(), 1, "{replies:?}");
14283 assert_eq!(replies[0].header.ty, FrameType::Error);
14284 let (_, module_rx) = self.modules.get_mut(target).unwrap();
14285 assert!(
14286 module_rx.try_recv().is_err(),
14287 "a refused open relays nothing"
14288 );
14289 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
14290 parse_error(&replies[0])
14291 }
14292
14293 async fn relayed(
14295 &mut self,
14296 opener: Option<&str>,
14297 target: &str,
14298 scope: Option<ScopeSelector>,
14299 ) -> Relayed {
14300 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
14301 let handler = self.handler.clone();
14302 let task_ctx = ctx.clone();
14303 let task = tokio::spawn(async move {
14304 handler
14305 .handle_control_frame(&task_ctx, frame)
14306 .await
14307 .unwrap()
14308 });
14309 let (_, module_rx) = self.modules.get_mut(target).unwrap();
14310 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
14311 .await
14312 .expect("the target receives the relayed route.bind")
14313 .unwrap()
14314 .frame;
14315 Relayed {
14316 target: target.to_string(),
14317 client: ctx,
14318 client_rx: rx,
14319 task,
14320 bind,
14321 }
14322 }
14323
14324 async fn ack(&self, relayed: &Relayed) {
14325 let (module, _) = &self.modules[&relayed.target];
14326 self.handler
14327 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
14328 .await
14329 .unwrap();
14330 }
14331
14332 async fn bound(
14334 &mut self,
14335 opener: Option<&str>,
14336 target: &str,
14337 scope: Option<ScopeSelector>,
14338 ) -> Bound {
14339 let relayed = self.relayed(opener, target, scope).await;
14340 self.ack(&relayed).await;
14341 let Relayed {
14342 target,
14343 client,
14344 mut client_rx,
14345 task,
14346 bind,
14347 } = relayed;
14348 assert!(
14349 task.await.unwrap().is_empty(),
14350 "the open is answered by commit"
14351 );
14352 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
14353 Bound {
14354 target,
14355 client,
14356 client_rx,
14357 channel,
14358 epoch,
14359 bind,
14360 }
14361 }
14362
14363 fn live(&self, route: &Bound) -> bool {
14364 matches!(
14365 self.forwarding
14366 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
14367 .unwrap(),
14368 DataRoute::Client(DataRouteState::Bound(_))
14369 )
14370 }
14371 }
14372
14373 struct Relayed {
14374 target: String,
14375 client: RouteCtx,
14376 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14377 task: tokio::task::JoinHandle<Vec<Frame>>,
14378 bind: Frame,
14379 }
14380
14381 struct Bound {
14382 target: String,
14383 client: RouteCtx,
14384 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14385 channel: u16,
14386 epoch: u32,
14387 bind: Frame,
14388 }
14389
14390 impl Bound {
14391 fn closed_reason(&mut self) -> RouteCloseReason {
14394 let mut reason = None;
14395 let mut goodbye = false;
14396 while let Ok(outbound) = self.client_rx.try_recv() {
14397 let frame = outbound.frame;
14398 match frame.header.ty {
14399 FrameType::Goodbye => {
14400 assert_eq!(
14401 (frame.header.channel, frame.header.epoch),
14402 (self.channel, self.epoch)
14403 );
14404 goodbye = true;
14405 }
14406 FrameType::Push => {
14407 let ClientControlPush::RouteClosed {
14408 reason: r,
14409 module_id,
14410 ..
14411 } = serde_json::from_slice(&frame.body).unwrap()
14412 else {
14413 panic!("unexpected push");
14414 };
14415 assert_eq!(module_id, self.target);
14416 reason = Some(r);
14417 }
14418 other => panic!("unexpected frame {other:?}"),
14419 }
14420 }
14421 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
14422 reason.expect("the client is told why the route closed")
14423 }
14424
14425 fn untouched(&mut self) -> bool {
14426 self.client_rx.try_recv().is_err()
14427 }
14428
14429 fn stamp(&self) -> Option<ScopeStamp> {
14430 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
14431 ModuleControlRequest::RouteBind { scope, .. } => scope,
14432 other => panic!("expected a route.bind, got {other:?}"),
14433 }
14434 }
14435 }
14436
14437 #[tokio::test]
14438 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
14439 ) {
14440 let mut rig = rig().await;
14441 let mut record = session(1);
14442 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
14443 record.child_owners = vec![Principal::Reserved {
14444 module_id: MAGIC.to_string(),
14445 }];
14446 rig.sync(vec![record]).await;
14447 let scope = || Some(rig_selector("s", Some(1)));
14448
14449 rig.bound(Some(OWNER), PLEXUS, scope()).await;
14452 rig.bound(Some(AFT), OTHER, scope()).await;
14453 rig.bound(Some(BROCA), PLEXUS, scope()).await;
14454
14455 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
14459 assert_eq!(
14460 rig.refused(opener, target, scope()).await,
14461 error_codes::SCOPE_NOT_CARRIER,
14462 "{opener:?} -> {target}"
14463 );
14464 }
14465 }
14466
14467 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
14468 ScopeSelector {
14469 owner: Principal::Reserved {
14470 module_id: OWNER.to_string(),
14471 },
14472 scope_ref: scope_ref.to_string(),
14473 scope_epoch,
14474 }
14475 }
14476
14477 #[tokio::test]
14478 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
14479 let mut rig = rig().await;
14480 rig.sync(vec![session(1)]).await;
14481 for opener in [OWNER, AFT] {
14482 assert_eq!(
14483 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
14484 .await,
14485 error_codes::SCOPE_EPOCH_REQUIRED,
14486 "{opener}"
14487 );
14488 }
14489 }
14490
14491 #[tokio::test]
14492 async fn admission_separates_not_synced_not_live_and_ended() {
14493 let mut rig = rig().await;
14494 let code = rig
14496 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14497 .await;
14498 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
14499 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
14500
14501 let ghost = ScopeSelector {
14503 owner: Principal::Reserved {
14504 module_id: "ghost".to_string(),
14505 },
14506 scope_ref: "s".to_string(),
14507 scope_epoch: Some(1),
14508 };
14509 assert_eq!(
14510 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
14511 error_codes::SCOPE_NOT_LIVE
14512 );
14513
14514 rig.sync(vec![session(2)]).await;
14515 assert_eq!(
14516 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
14517 .await,
14518 error_codes::SCOPE_NOT_LIVE
14519 );
14520 for epoch in [1, 3] {
14521 assert_eq!(
14522 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
14523 .await,
14524 error_codes::SCOPE_ENDED,
14525 "epoch {epoch}"
14526 );
14527 }
14528 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
14530 .await;
14531 }
14532
14533 #[tokio::test]
14534 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
14535 let mut rig = rig().await;
14536 rig.sync(vec![session(1)]).await;
14537 let route = rig
14538 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14539 .await;
14540 let stamp = route.stamp().expect("a scoped bind carries the stamp");
14541 assert_eq!(stamp.scope_ref, "s");
14542 assert_eq!(stamp.scope_epoch, 1);
14543 assert_eq!(stamp.kind, ScopeKind::Head);
14544 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
14545 assert!(stamp.attributes.delegates);
14546 assert!(stamp.owner_authorized);
14547 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14548 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
14549
14550 let (broca, mut broca_rx) = wide_ctx(50);
14553 hello_via_sink(
14554 &rig.handler,
14555 &broca,
14556 &mut broca_rx,
14557 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
14558 )
14559 .await;
14560 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
14561 .await
14562 .unwrap();
14563 let own = ScopeSelector {
14564 owner: Principal::Reserved {
14565 module_id: BROCA.to_string(),
14566 },
14567 scope_ref: "b".to_string(),
14568 scope_epoch: Some(1),
14569 };
14570 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
14571 assert!(!route.stamp().unwrap().owner_authorized);
14572 }
14573
14574 #[tokio::test]
14575 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
14576 let mut rig = rig().await;
14577 let mut record = head("s", 1);
14578 record.carriers = vec![carrier(AFT, None)];
14579 let flow_id = "Flow:run-7/step_2!~";
14580 record.attributes.flow_id = Some(flow_id.to_string());
14581 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
14582 .await
14583 .unwrap();
14584 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
14585 panic!("not a sync reply");
14586 };
14587 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
14588 let route = rig
14589 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14590 .await;
14591 assert!(rig.live(&route), "the stamped bind committed");
14592 let stamp = route.stamp().expect("a flow scope carries a stamp");
14593 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
14594 assert_eq!(stamp.attributes.agent_id, None);
14595 assert!(!stamp.attributes.delegates);
14596 assert!(stamp.owner_authorized);
14597 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14598 assert_eq!(unscoped.stamp(), None);
14599 }
14600
14601 #[tokio::test]
14602 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
14603 let mut rig = rig_with_flow_support(false).await;
14604 let mut record = session(1);
14605 record.attributes.flow_id = Some("flow:7".to_string());
14606 rig.sync(vec![record]).await;
14607 for opener in [OWNER, AFT] {
14608 let body = rig
14609 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
14610 .await;
14611 assert_eq!(body["code"], "target_flow_unsupported");
14612 let message = body["message"].as_str().unwrap();
14613 for required in [PLEXUS, "flow-scopes/v1"] {
14614 assert!(message.contains(required), "{message}");
14615 }
14616 }
14617 }
14618
14619 #[tokio::test]
14620 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
14621 let mut rig = rig_with_flow_support(true).await;
14622 let mut record = session(1);
14623 record.attributes.flow_id = Some("flow:7".to_string());
14624 rig.sync(vec![record]).await;
14625 for opener in [OWNER, AFT] {
14626 let route = rig
14627 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
14628 .await;
14629 assert!(rig.live(&route));
14630 assert_eq!(
14631 route.stamp().unwrap().attributes.flow_id.as_deref(),
14632 Some("flow:7")
14633 );
14634 }
14635 }
14636
14637 #[tokio::test]
14638 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
14639 use std::future::Future;
14640
14641 let mut rig = rig_with_flow_support(true).await;
14642 let mut record = session(1);
14643 record.attributes.flow_id = Some("flow:7".to_string());
14644 rig.sync(vec![record]).await;
14645 let (client, mut client_rx, frame) =
14646 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
14647 for _ in 0..64 {
14650 client.egress.try_send(route_bind_ack(1)).unwrap();
14651 }
14652 let handler = rig.handler.clone();
14653 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
14654 std::future::poll_fn(|cx| {
14655 assert!(open.as_mut().poll(cx).is_pending());
14656 std::task::Poll::Ready(())
14657 })
14658 .await;
14659 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14660
14661 let old_connection = rig.modules[PLEXUS].0.connection_id;
14662 rig.handler.cleanup_connection(old_connection).unwrap();
14663 let (replacement, mut replacement_rx) = wide_ctx(200);
14664 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
14665 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
14666 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
14667 let hello = Frame::build(
14668 FrameType::Hello,
14669 control_flags(),
14670 0,
14671 0,
14672 200,
14673 serde_json::to_vec(&body).unwrap(),
14674 )
14675 .unwrap();
14676 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
14677
14678 client_rx.try_recv().unwrap();
14679 let replies = tokio::time::timeout(Duration::from_secs(2), open)
14680 .await
14681 .expect("the replacement is refused without waiting for a bind ack")
14682 .unwrap();
14683 assert_eq!(replies.len(), 1);
14684 let body = parse_error(&replies[0]);
14685 assert_eq!(body["code"], "target_flow_unsupported");
14686 for required in [PLEXUS, "flow-scopes/v1"] {
14687 assert!(body["message"].as_str().unwrap().contains(required));
14688 }
14689 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
14690 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
14691 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14692 assert!(rig
14693 .handler
14694 .registry
14695 .get_module_by_connection(replacement.connection_id)
14696 .unwrap()
14697 .is_some());
14698 }
14699
14700 #[tokio::test]
14701 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
14702 {
14703 let mut rig = rig_with_flow_support(false).await;
14704 rig.sync(vec![session(1)]).await;
14705 let route = rig
14706 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14707 .await;
14708 assert!(rig.live(&route));
14709 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
14710 let mut record = session(1);
14711 record.attributes.flow_id = Some("flow:7".to_string());
14712 rig.sync(vec![record]).await;
14713 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
14714 assert!(rig.live(&unscoped));
14715 assert_eq!(unscoped.stamp(), None);
14716 }
14717
14718 #[tokio::test]
14719 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
14720 let mut rig = rig().await;
14721 let mut record = session(1);
14722 record.attributes.flow_id = Some("flow:7".to_string());
14723 rig.sync(vec![record.clone()]).await;
14724 let before = rig.forwarding.published_scope_tag(OWNER, "s");
14725 let mut owner_route = rig
14726 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
14727 .await;
14728 let mut carrier_route = rig
14729 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
14730 .await;
14731 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14732 rig.sync(vec![record.clone()]).await;
14733 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
14734 assert!(rig.live(&owner_route) && owner_route.untouched());
14735 assert!(rig.live(&carrier_route) && carrier_route.untouched());
14736
14737 record.attributes.flow_id = Some("flow:8".to_string());
14738 rig.sync(vec![record]).await;
14739 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
14740 let before = before.unwrap();
14741 assert_eq!(after.scope_epoch, before.scope_epoch);
14742 assert!(after.version > before.version);
14743 for route in [&mut owner_route, &mut carrier_route] {
14744 assert!(!rig.live(route));
14745 assert_eq!(
14746 route.closed_reason(),
14747 RouteCloseReason::ScopeDelegationChanged
14748 );
14749 }
14750 assert!(rig.live(&unscoped) && unscoped.untouched());
14751 for target in [PLEXUS, OTHER] {
14754 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
14755 let goodbye = module_rx
14756 .try_recv()
14757 .expect("the provider sees the drain")
14758 .frame;
14759 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14760 assert!(module_rx.try_recv().is_err());
14761 }
14762 let rebound = rig
14763 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14764 .await;
14765 assert_eq!(
14766 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
14767 Some("flow:8")
14768 );
14769 }
14770
14771 #[tokio::test]
14776 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
14777 let mut rig = rig().await;
14778 rig.sync(vec![session(1)]).await;
14779 let mut cotenant = rig
14780 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
14781 .await;
14782
14783 let mut changed = session(1);
14784 changed.child_owners.push(Principal::Reserved {
14785 module_id: MAGIC.to_string(),
14786 });
14787 let mut ended = None;
14788 for (code, next) in [
14789 (error_codes::SCOPE_CHANGED, vec![changed]),
14790 (error_codes::SCOPE_ENDED, Vec::new()),
14791 ] {
14792 let relayed = rig
14793 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14794 .await;
14795 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
14796 ended = Some(next.is_empty());
14797 rig.sync(next).await;
14798 rig.ack(&relayed).await;
14799 let replies = relayed.task.await.unwrap();
14800 assert_eq!(replies.len(), 1, "{replies:?}");
14801 assert_eq!(parse_error(&replies[0])["code"], code);
14802 assert_eq!(
14803 subc_protocol::error_codes::is_retryable_route_open(code),
14804 code == error_codes::SCOPE_CHANGED
14805 );
14806 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
14807 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
14809 let mut goodbyes = Vec::new();
14812 while let Ok(outbound) = plexus_rx.try_recv() {
14813 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
14814 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
14815 }
14816 assert!(
14817 goodbyes.contains(&(bind_channel, bind_epoch)),
14818 "{goodbyes:?}"
14819 );
14820 assert!(rig
14821 .handler
14822 .registry
14823 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
14824 .unwrap()
14825 .is_some());
14826 }
14827 assert_eq!(ended, Some(true));
14828 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
14831 }
14832
14833 #[tokio::test]
14836 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
14837 struct Case {
14838 name: &'static str,
14839 change: fn(&mut ScopeRecord),
14840 closed: [Option<RouteCloseReason>; 4],
14843 }
14844 use RouteCloseReason::*;
14845 let cases = [
14846 Case {
14847 name: "a carrier entry removed",
14848 change: |r| {
14849 r.carriers.retain(|c| {
14850 c.principal
14851 != Principal::Reserved {
14852 module_id: AFT.to_string(),
14853 }
14854 })
14855 },
14856 closed: [None, Some(ScopeCarrierRemoved), None, None],
14857 },
14858 Case {
14859 name: "a target removed from a carrier",
14860 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
14861 closed: [None, None, None, Some(ScopeCarrierRemoved)],
14862 },
14863 Case {
14864 name: "a bare carrier narrowed to targets",
14865 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
14866 closed: [None, Some(ScopeCarrierRemoved), None, None],
14867 },
14868 Case {
14869 name: "delegates turned off",
14870 change: |r| r.attributes.delegates = false,
14871 closed: [Some(ScopeDelegationChanged); 4],
14872 },
14873 Case {
14874 name: "agent_id changed",
14875 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
14876 closed: [Some(ScopeDelegationChanged); 4],
14877 },
14878 Case {
14879 name: "a carrier added, child owners changed, the record re-sent",
14880 change: |r| {
14881 r.carriers.push(carrier(MAGIC, None));
14882 r.child_owners.push(Principal::Reserved {
14883 module_id: MAGIC.to_string(),
14884 });
14885 },
14886 closed: [None; 4],
14887 },
14888 Case {
14889 name: "a target added",
14890 change: |r| {
14891 r.carriers[1]
14892 .targets
14893 .as_mut()
14894 .unwrap()
14895 .push("third".to_string())
14896 },
14897 closed: [None; 4],
14898 },
14899 Case {
14900 name: "delegates turned on",
14901 change: |r| r.attributes.delegates = true,
14902 closed: [None; 4],
14903 },
14904 ];
14905 for case in cases {
14906 let mut rig = rig().await;
14907 rig.sync(vec![session(1)]).await;
14908 let scope = || Some(rig_selector("s", Some(1)));
14909 let mut routes = [
14910 rig.bound(Some(OWNER), PLEXUS, scope()).await,
14911 rig.bound(Some(AFT), PLEXUS, scope()).await,
14912 rig.bound(Some(BROCA), PLEXUS, scope()).await,
14913 rig.bound(Some(BROCA), OTHER, scope()).await,
14914 ];
14915 let mut record = session(1);
14916 (case.change)(&mut record);
14917 rig.sync(vec![record]).await;
14918 for (index, expected) in case.closed.iter().enumerate() {
14919 let route = &mut routes[index];
14920 match expected {
14921 Some(reason) => {
14922 assert!(!rig.live(route), "{}: route {index} still live", case.name);
14923 assert_eq!(
14924 route.closed_reason(),
14925 *reason,
14926 "{}: route {index}",
14927 case.name
14928 );
14929 }
14930 None => {
14931 assert!(rig.live(route), "{}: route {index} closed", case.name);
14932 assert!(
14933 route.untouched(),
14934 "{}: route {index} was told something",
14935 case.name
14936 );
14937 }
14938 }
14939 }
14940 }
14941 }
14942
14943 #[tokio::test]
14944 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
14945 for next in [Vec::new(), vec![session(2)]] {
14947 let mut rig = rig().await;
14948 rig.sync(vec![session(1)]).await;
14949 let mut route = rig
14950 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14951 .await;
14952 rig.sync(next).await;
14953 assert!(!rig.live(&route));
14954 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
14955 }
14956
14957 let mut rig = rig().await;
14960 let mut child = session(1);
14961 child.scope_ref = "child".to_string();
14962 child.kind = ScopeKind::Worker;
14963 child.parent = Some(ScopeParent::new(
14964 Principal::Reserved {
14965 module_id: OWNER.to_string(),
14966 },
14967 "s".to_string(),
14968 1,
14969 ));
14970 rig.sync(vec![session(1), child.clone()]).await;
14971 let mut child_route = rig
14972 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
14973 .await;
14974 assert_eq!(
14975 child_route.stamp().unwrap().parent_state,
14976 Some(ParentState::Linked)
14977 );
14978 rig.sync(vec![child]).await;
14979 assert!(!rig.live(&child_route));
14980 assert_eq!(
14981 child_route.closed_reason(),
14982 RouteCloseReason::ScopeParentEnded
14983 );
14984 }
14985
14986 #[tokio::test]
14987 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
14988 ) {
14989 let mut rig = rig().await;
14990 rig.sync(vec![session(1)]).await;
14991 let mut route = rig
14992 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14993 .await;
14994 let before = rig.forwarding.published_scope_tag(OWNER, "s");
14995 rig.sync(vec![session(1)]).await;
14996 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
14997 assert!(rig.live(&route) && route.untouched());
14998
14999 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
15003 .forwarding
15004 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15005 .unwrap()
15006 else {
15007 panic!("the route is bound");
15008 };
15009 binding.flow.acquire_tagged(9, false).await.unwrap();
15010 let mut widened = session(1);
15011 widened.carriers.push(carrier(MAGIC, None));
15012 rig.sync(vec![widened]).await;
15013 assert!(rig.live(&route) && route.untouched());
15014 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15015 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
15016 assert_eq!(binding.flow.in_flight(), 1);
15019 binding
15020 .flow
15021 .acquire_tagged(10, false)
15022 .await
15023 .expect("the flow is still open");
15024 }
15025
15026 #[tokio::test]
15029 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
15030 let mut rig = rig().await;
15031 rig.sync(vec![session(1)]).await;
15032 let mut on_incumbent = rig
15033 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15034 .await;
15035
15036 let (candidate, _candidate_rx) = wide_ctx(9);
15039 let registration = rig
15040 .handler
15041 .registry
15042 .register_candidate_with_control_ops(
15043 manifest(PLEXUS, PROTOCOL_VERSION),
15044 PROTOCOL_VERSION,
15045 candidate.connection_id,
15046 module_baseline_control_ops(),
15047 )
15048 .unwrap();
15049 rig.forwarding
15050 .register_candidate_module_connection(
15051 candidate.connection_id,
15052 PLEXUS.to_string(),
15053 PROTOCOL_VERSION,
15054 manifest_concurrency(®istration.manifest),
15055 candidate.egress.clone(),
15056 )
15057 .unwrap();
15058 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
15059 rig.handler
15060 .registry
15061 .promote_candidate(PLEXUS)
15062 .unwrap()
15063 .unwrap();
15064 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
15065
15066 rig.sync(Vec::new()).await;
15067 assert!(!rig.live(&on_incumbent));
15068 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
15069 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15070 let goodbye = incumbent_rx
15071 .try_recv()
15072 .expect("the superseded endpoint is told")
15073 .frame;
15074 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15075 }
15076 }
15077}
15078
15079#[cfg(test)]
15080mod concurrency_default_exposure_tests {
15081 use super::*;
15082
15083 fn hello_body(role_json: &str) -> Vec<u8> {
15084 format!(
15085 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
15086 )
15087 .into_bytes()
15088 }
15089
15090 fn manifest_from(body: &[u8]) -> ModuleManifest {
15091 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
15092 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
15093 .expect("manifest parses")
15094 }
15095
15096 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
15097
15098 #[test]
15099 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
15100 let body = hello_body(&format!(
15101 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
15102 ));
15103 let manifest = manifest_from(&body);
15104 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
15107 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
15108 }
15109
15110 #[test]
15111 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
15112 let body = hello_body(&format!(
15113 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
15114 ));
15115 let manifest = manifest_from(&body);
15116 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
15117 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
15118 }
15119
15120 #[test]
15121 fn non_management_roles_are_never_reported() {
15122 let body = hello_body(
15123 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
15124 );
15125 let manifest = manifest_from(&body);
15126 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
15127 }
15128}