1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 path::{Path, PathBuf},
5 sync::{Arc, Mutex, RwLock},
6 time::{Duration, Instant as StdInstant},
7};
8
9use serde::{Deserialize, Serialize};
10use subc_control::{
11 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
12 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
13 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
14 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
15 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
16 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
17 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
18};
19use subc_protocol::{
20 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
21 error_codes,
22 manifest::{
23 validate_hello_capability_grammar, validate_hello_event_declarations,
24 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
25 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
26 },
27 scope::{
28 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
29 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
30 SCOPE_SYNC_OP,
31 },
32 session::{
33 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
34 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
35 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
36 ROLE_VERSIONS_FIELD,
37 },
38 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
39 Priority, RouteTarget, PROTOCOL_VERSION,
40};
41use tokio::time::{timeout_at, Instant};
42use tracing::{debug, info, warn};
43
44use crate::{
45 capability_requirements::{
46 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
47 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
48 RuntimeModule,
49 },
50 daemon_config::RestartRequiredSection,
51 forwarding::{
52 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
53 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
54 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
55 },
56 observability::{
57 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
58 },
59 provenance::{
60 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
61 },
62 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
63 router::{RouteCtx, RouterError},
64 scopes::{BoundScope, HelloLaunchNonces, ScopeTable},
65 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
66 stderr_tail::{CaptureState, TailEntry},
67 supervise::{
68 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
69 SupervisorHandle, SwapHelloAdmission,
70 },
71 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
72};
73
74pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
80
81const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
82const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
83const CAP_PING_PONG: &str = "ping_pong_v1";
84const CAP_SESSION_ATTACH: &str = "session_attach_v1";
85const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
86
87const SUBC_CONTROL_OPS: &[&str] = &[
88 ops::ACTIVITY_RECORD,
89 ops::ACTIVITY_QUERY,
90 ops::ACTIVITY_SUBSCRIBE,
91 ops::SERVER_DESCRIBE,
92 ops::CATALOG_LIST,
93 ops::ROUTE_OPEN,
94 ops::ROUTE_POLL,
95 ops::ROUTE_CLOSING,
96 ops::ROUTE_CLOSED,
97 ops::SUPERVISOR_LIST,
98 ops::SUPERVISOR_RESTART,
99 ops::SUPERVISOR_SWAP,
100 ops::SUPERVISOR_RELOAD,
101 ops::SUPERVISOR_RESCAN,
102 ops::SUPERVISOR_RELEASE_RESERVED,
103 ops::SUPERVISOR_SET_ENABLED,
104 ops::SUPERVISOR_HEALTH_PROBE,
105 ops::SUPERVISOR_HEALTH,
106 ops::SUPERVISOR_STDERR_TAIL,
107 ops::SUPERVISOR_TERMINALS,
108 ops::SUPERVISOR_ROUTES,
109 ops::SUPERVISOR_PROVENANCE,
110 ops::SUPERVISOR_SPAWN_SNAPSHOT,
111 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
112];
113
114const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
115 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
116 "supervisor.live_roots",
117 SCOPE_SYNC_OP,
118 SCOPE_APPLY_OP,
119 SCOPE_DESCRIBE_OP,
120 "operator.confirm",
121];
122
123const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
127
128const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
129
130pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
138
139pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
154
155pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
173
174const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
175const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
176
177fn reload_verdict(
178 configured: &Path,
179 spawned_from: Option<&Path>,
180 image: subc_control::RunningImageAgreement,
181) -> PendingReloadVerdict {
182 let path = match spawned_from {
183 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
184 ReloadPathAgreement::Match
185 }
186 Some(spawned_from) => ReloadPathAgreement::Mismatch {
187 configured: configured.to_path_buf(),
188 spawned_from: spawned_from.to_path_buf(),
189 },
190 None => ReloadPathAgreement::Unavailable {
191 reason: if matches!(
192 image,
193 subc_control::RunningImageAgreement::Unavailable {
194 reason: subc_control::RunningImageUnavailableReason::NotRunning
195 }
196 ) {
197 ReloadPathUnavailableReason::NotRunning
198 } else {
199 ReloadPathUnavailableReason::SpawnedPathUnavailable
200 },
201 },
202 };
203 PendingReloadVerdict { path, image }
204}
205
206fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
207 if configured == spawned_from {
208 return true;
209 }
210 #[cfg(windows)]
211 {
212 match (
218 std::fs::canonicalize(configured),
219 std::fs::canonicalize(spawned_from),
220 ) {
221 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
222 _ => false,
223 }
224 }
225 #[cfg(not(windows))]
226 {
227 false
228 }
229}
230
231#[derive(Clone)]
232struct DaemonProvenanceFacts {
233 build: DaemonBuildProvenance,
234 pid: Option<u32>,
235 started_at_ms: Option<u64>,
236 start_clock: Option<crate::clock::StartClock>,
237 executable_path: Option<PathBuf>,
238 executable_identity: Option<SpawnedFileIdentity>,
239 process_start_time: Option<u64>,
240 probe: ExecutableIdentityProbe,
241}
242
243impl Default for DaemonProvenanceFacts {
244 fn default() -> Self {
245 Self {
246 build: DaemonBuildProvenance {
247 build_git_sha: None,
248 build_lock_digest: None,
249 },
250 pid: None,
251 started_at_ms: None,
252 start_clock: None,
253 executable_path: None,
254 executable_identity: None,
255 process_start_time: None,
256 probe: ExecutableIdentityProbe::default(),
257 }
258 }
259}
260
261#[derive(Debug, Clone)]
262struct SupervisorRescanContext {
263 supervisor: Supervisor,
264 config_path: PathBuf,
265 configured_port: Option<u16>,
266 storage_config: Option<crate::daemon_config::StorageConfig>,
267 admission_facts_carrier_module_id: Option<String>,
268 admission_facts_targets: Option<Vec<String>>,
269 scope_authority_owners: Vec<String>,
270}
271
272const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
293 "reloading",
294 "supervisor_not_live",
295 "registration_not_active",
296 "no_forwarding_connection",
297 "relay_send_failed",
298];
299
300#[derive(Clone)]
302pub struct ControlHandler {
303 registry: Arc<Registry>,
304 activity: Option<crate::activity::ActivityLedger>,
305 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
306 forwarding: Arc<ForwardingTable>,
307 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
308 supervisor: SupervisorHandle,
309 subc_capabilities: Arc<[String]>,
310 route_bind_relay_timeout: Duration,
314 route_bind_relay_timeouts: BTreeMap<String, Duration>,
318 route_bind_breakers: RouteBindBreakers,
321 route_bind_concurrency: RouteBindConcurrency,
324 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
329 route_bind_breaker_threshold: u32,
331 route_bind_breaker_cooldown: Duration,
333 health_probe_timeout: Duration,
334 storage_config: Option<crate::daemon_config::StorageConfig>,
337 machine_id: Option<crate::machine_id::MachineId>,
341 admission_facts_carrier_module_id: Option<String>,
342 admission_facts_targets: Option<Vec<String>>,
343 scopes: Arc<RwLock<ScopeTable>>,
347 scope_deadline_added: Arc<tokio::sync::Notify>,
351 scope_authority_owners: Vec<String>,
354 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
357 rescan: Option<SupervisorRescanContext>,
358 connected_clients: ConnectedClients,
359 counters: DaemonCounters,
360 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
361 daemon_provenance: DaemonProvenanceFacts,
362 #[cfg(test)]
363 control_dispatch_delay: Option<Duration>,
364 #[cfg(test)]
365 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
366 #[cfg(test)]
370 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
371 #[cfg(test)]
377 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
378}
379
380impl fmt::Debug for ControlHandler {
381 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
382 f.debug_struct("ControlHandler")
383 .field("registry", &self.registry)
384 .field("forwarding", &self.forwarding)
385 .field("process_liveness", &self.process_liveness.is_some())
386 .field("supervisor", &self.supervisor)
387 .field("subc_capabilities", &self.subc_capabilities)
388 .finish()
389 }
390}
391
392struct RouteOpenRequest {
393 target: RouteTarget,
394 identity: BindIdentity,
395 consumer_identity: Option<ConsumerIdentity>,
396 consumer_capabilities: Option<Vec<String>>,
397 role_versions: Option<BTreeMap<String, String>>,
398 admission_facts: Option<serde_json::Value>,
399 scope: Option<ScopeSelector>,
400}
401
402struct RouteBindReservationGuard {
403 forwarding: Arc<ForwardingTable>,
404 endpoint: ModuleEndpointId,
405 relay_corr: u64,
406 armed: bool,
407}
408
409struct ModuleControlRpcGuard {
410 forwarding: Arc<ForwardingTable>,
411 endpoint: ModuleEndpointId,
412 corr: u64,
413 armed: bool,
414}
415
416impl ModuleControlRpcGuard {
417 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
418 Self {
419 forwarding,
420 endpoint,
421 corr,
422 armed: true,
423 }
424 }
425
426 fn disarm(&mut self) {
427 self.armed = false;
428 }
429}
430
431impl Drop for ModuleControlRpcGuard {
432 fn drop(&mut self) {
433 if self.armed {
434 let _ = self
435 .forwarding
436 .cancel_module_control_rpc(self.endpoint, self.corr);
437 }
438 }
439}
440
441impl RouteBindReservationGuard {
442 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
443 Self {
444 forwarding,
445 endpoint,
446 relay_corr,
447 armed: true,
448 }
449 }
450
451 fn release_and_disarm(&mut self) {
452 if !self.armed {
453 return;
454 }
455 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
456 self.endpoint,
457 self.relay_corr,
458 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
459 ) {
460 send_goodbye_target_best_effort(
461 &self.forwarding.counters(),
462 &target,
463 "canceled route.bind",
464 );
465 }
466 self.armed = false;
467 }
468
469 fn disarm(&mut self) {
470 self.armed = false;
471 }
472}
473
474impl Drop for RouteBindReservationGuard {
475 fn drop(&mut self) {
476 self.release_and_disarm();
477 }
478}
479
480#[derive(Debug, Clone, Default)]
510pub(crate) struct RouteBindBreakers {
511 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
512}
513
514#[derive(Debug, Clone, Default)]
515pub(crate) struct RouteBindConcurrency {
516 modules: Arc<Mutex<HashMap<String, usize>>>,
517}
518
519struct RouteBindConcurrencyGuard {
520 concurrency: RouteBindConcurrency,
521 module_id: String,
522}
523
524impl RouteBindConcurrency {
525 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
528 let mut modules = self
529 .modules
530 .lock()
531 .expect("route.bind concurrency mutex poisoned");
532 let in_flight = modules.entry(module_id.to_string()).or_default();
533 if *in_flight >= limit {
534 return Err(*in_flight);
535 }
536 *in_flight += 1;
537 Ok(RouteBindConcurrencyGuard {
538 concurrency: self.clone(),
539 module_id: module_id.to_string(),
540 })
541 }
542}
543
544impl Drop for RouteBindConcurrencyGuard {
545 fn drop(&mut self) {
546 let mut modules = self
547 .concurrency
548 .modules
549 .lock()
550 .expect("route.bind concurrency mutex poisoned");
551 let remove = {
552 let in_flight = modules
553 .get_mut(&self.module_id)
554 .expect("admitted route.bind has a concurrency entry");
555 *in_flight -= 1;
556 *in_flight == 0
557 };
558 if remove {
559 modules.remove(&self.module_id);
560 }
561 }
562}
563
564#[derive(Debug, Default)]
565struct ModuleBreakerState {
566 consecutive_timeouts: u32,
568 cooldown_until: Option<Instant>,
571 probe_in_flight: Option<Arc<()>>,
576}
577
578enum RouteBindAdmission<'a> {
580 Admitted {
581 guard: RouteBindBreakerGuard<'a>,
582 probe: bool,
585 },
586 Refused {
587 consecutive_timeouts: u32,
588 retry_in: Duration,
592 probe_in_flight: bool,
593 },
594}
595
596struct RouteBindBreakerGuard<'a> {
603 breakers: RouteBindBreakers,
604 module_id: &'a str,
605 probe_token: Option<Arc<()>>,
606 settled: bool,
607}
608
609impl RouteBindBreakerGuard<'_> {
610 fn record_accepted(&mut self) -> bool {
614 self.settled = true;
615 self.breakers.record_accepted(self.module_id)
616 }
617
618 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
621 self.settled = true;
622 self.breakers.record_timeout(
623 self.module_id,
624 self.probe_token.as_ref(),
625 threshold,
626 cooldown,
627 )
628 }
629
630 fn record_inconclusive(&mut self) {
639 self.settled = true;
640 self.breakers
641 .record_inconclusive(self.module_id, self.probe_token.as_ref());
642 }
643}
644
645impl Drop for RouteBindBreakerGuard<'_> {
646 fn drop(&mut self) {
647 if !self.settled {
648 self.breakers
649 .record_inconclusive(self.module_id, self.probe_token.as_ref());
650 }
651 }
652}
653
654struct BreakerOpened {
658 consecutive_timeouts: u32,
659 reopened_after_probe: bool,
662}
663
664impl RouteBindBreakers {
665 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
666 self.modules
667 .lock()
668 .expect("route.bind breaker mutex poisoned")
669 }
670
671 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
674 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
675 probe: probe_token.is_some(),
676 guard: RouteBindBreakerGuard {
677 breakers: self.clone(),
678 module_id,
679 probe_token,
680 settled: false,
681 },
682 };
683
684 let mut modules = self.lock();
685 let Some(state) = modules.get_mut(module_id) else {
686 return admitted(None);
687 };
688 let Some(cooldown_until) = state.cooldown_until else {
689 return admitted(None);
690 };
691 if state.probe_in_flight.is_some() {
692 return RouteBindAdmission::Refused {
693 consecutive_timeouts: state.consecutive_timeouts,
694 retry_in: Duration::ZERO,
695 probe_in_flight: true,
696 };
697 }
698 let now = Instant::now();
699 if now < cooldown_until {
700 return RouteBindAdmission::Refused {
701 consecutive_timeouts: state.consecutive_timeouts,
702 retry_in: cooldown_until - now,
703 probe_in_flight: false,
704 };
705 }
706 let token = Arc::new(());
707 state.probe_in_flight = Some(Arc::clone(&token));
708 admitted(Some(token))
709 }
710
711 fn record_accepted(&self, module_id: &str) -> bool {
712 self.lock()
713 .remove(module_id)
714 .is_some_and(|state| state.cooldown_until.is_some())
715 }
716
717 fn record_timeout(
718 &self,
719 module_id: &str,
720 probe_token: Option<&Arc<()>>,
721 threshold: u32,
722 cooldown: Duration,
723 ) -> Option<BreakerOpened> {
724 let mut modules = self.lock();
725 let state = modules.entry(module_id.to_string()).or_default();
726 let was_open = state.cooldown_until.is_some();
727 let was_probe = Self::owns_probe(state, probe_token);
728 if was_probe {
729 state.probe_in_flight = None;
730 }
731 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
732 if state.consecutive_timeouts < threshold {
733 return None;
734 }
735 state.cooldown_until = Some(Instant::now() + cooldown);
736 Some(BreakerOpened {
737 consecutive_timeouts: state.consecutive_timeouts,
738 reopened_after_probe: was_open && was_probe,
739 })
740 }
741
742 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
743 state
747 .probe_in_flight
748 .as_ref()
749 .zip(token)
750 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
751 }
752
753 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
754 if let Some(state) = self.lock().get_mut(module_id) {
755 if Self::owns_probe(state, probe_token) {
756 state.probe_in_flight = None;
757 }
758 }
759 }
760
761 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
776 self.lock()
777 .remove(module_id)
778 .map(|state| state.consecutive_timeouts)
779 .filter(|discarded| *discarded > 0)
780 }
781
782 fn open_snapshot(&self) -> Option<serde_json::Value> {
789 let now = Instant::now();
790 let modules = self.lock();
791 let open = modules
792 .iter()
793 .filter_map(|(module_id, state)| {
794 let cooldown_until = state.cooldown_until?;
795 Some((
796 module_id.clone(),
797 serde_json::json!({
798 "consecutive_timeouts": state.consecutive_timeouts,
799 "cooldown_remaining_ms":
800 cooldown_until.saturating_duration_since(now).as_millis() as u64,
801 "probe_in_flight": state.probe_in_flight.is_some(),
802 }),
803 ))
804 })
805 .collect::<serde_json::Map<String, serde_json::Value>>();
806 (!open.is_empty()).then_some(serde_json::Value::Object(open))
807 }
808}
809
810impl ControlHandler {
811 pub fn new(registry: Arc<Registry>) -> Self {
812 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
813 }
814
815 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
816 let counters = forwarding.counters();
817 let route_bind_breakers = forwarding.route_bind_breakers();
821 let route_bind_concurrency = forwarding.route_bind_concurrency();
822 let route_outages = forwarding.route_outages();
823 Self {
824 registry,
825 activity: None,
826 activity_streams: Arc::new(Mutex::new(HashMap::new())),
827 forwarding,
828 process_liveness: None,
829 supervisor: SupervisorHandle::new(),
830 subc_capabilities: Arc::from([
831 CAP_MANIFEST_REGISTRATION.to_string(),
832 CAP_CHANNEL_LIFECYCLE.to_string(),
833 CAP_PING_PONG.to_string(),
834 CAP_SESSION_ATTACH.to_string(),
835 CAP_ADMISSION_FACTS_RELAY.to_string(),
836 CAP_SCOPES_V1.to_string(),
837 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
838 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
839 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
840 ]),
841 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
842 route_bind_relay_timeouts: BTreeMap::new(),
843 route_bind_breakers,
844 route_bind_concurrency,
845 route_outages,
846 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
847 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
848 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
849 storage_config: None,
850 machine_id: None,
851 admission_facts_carrier_module_id: None,
852 admission_facts_targets: None,
853 scopes: Arc::new(RwLock::new(ScopeTable::new(
854 crate::daemon_config::default_scope_authority_owners(),
855 ))),
856 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
857 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
858 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
859 rescan: None,
860 connected_clients: ConnectedClients::new(),
861 counters,
862 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
863 daemon_provenance: DaemonProvenanceFacts::default(),
864 #[cfg(test)]
865 control_dispatch_delay: None,
866 #[cfg(test)]
867 provenance_probe_override: None,
868 #[cfg(test)]
869 route_bind_relay_sent_hold: None,
870 #[cfg(test)]
871 hello_registry_to_forwarding_hold: None,
872 }
873 }
874
875 pub fn with_storage_config(
878 mut self,
879 storage_config: Option<crate::daemon_config::StorageConfig>,
880 ) -> Self {
881 self.storage_config = storage_config;
882 self
883 }
884
885 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
888 self.activity = Some(crate::activity::ActivityLedger::open(
889 run_dir.join("activity.jsonl"),
890 incarnation,
891 ));
892 self.supervisor.set_activity(self.activity.clone());
893 self
894 }
895
896 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
899 self.machine_id = machine_id;
900 self
901 }
902
903 pub fn with_admission_facts_config(
907 mut self,
908 carrier_module_id: Option<String>,
909 targets: Option<Vec<String>>,
910 ) -> Self {
911 self.admission_facts_carrier_module_id = carrier_module_id;
912 self.admission_facts_targets = targets;
913 self
914 }
915
916 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
920 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
921 self.scope_authority_owners = owners;
922 self
923 }
924
925 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
928 self.route_bind_relay_timeout = timeout;
929 self
930 }
931
932 pub fn with_route_bind_relay_timeouts(
938 mut self,
939 timeouts: impl IntoIterator<Item = (String, Duration)>,
940 ) -> Self {
941 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
942 self
943 }
944
945 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
951 self.route_bind_relay_timeouts
952 .get(module_id)
953 .copied()
954 .unwrap_or(self.route_bind_relay_timeout)
955 }
956
957 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
965 self.route_bind_breaker_threshold = threshold.max(1);
966 self.route_bind_breaker_cooldown = cooldown;
967 self
968 }
969
970 #[cfg(test)]
971 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
972 self.health_probe_timeout = timeout;
973 self
974 }
975
976 #[cfg(test)]
977 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
978 self.control_dispatch_delay = Some(delay);
979 self
980 }
981
982 #[cfg(test)]
986 pub(crate) fn with_route_bind_relay_sent_hold(
987 mut self,
988 hold: Arc<tokio::sync::Barrier>,
989 ) -> Self {
990 self.route_bind_relay_sent_hold = Some(hold);
991 self
992 }
993
994 #[cfg(test)]
999 pub(crate) fn with_hello_registry_to_forwarding_hold(
1000 mut self,
1001 hold: Arc<std::sync::Barrier>,
1002 ) -> Self {
1003 self.hello_registry_to_forwarding_hold = Some(hold);
1004 self
1005 }
1006
1007 #[cfg(test)]
1008 fn hold_between_registry_and_forwarding(&self) {
1009 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1010 hold.wait();
1011 hold.wait();
1012 }
1013 }
1014
1015 pub fn with_process_liveness(
1016 mut self,
1017 process_liveness: Arc<dyn ModuleProcessLiveness>,
1018 ) -> Self {
1019 self.process_liveness = Some(process_liveness);
1020 self
1021 }
1022
1023 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1024 self.supervisor = supervisor;
1025 self.supervisor.set_activity(self.activity.clone());
1026 self
1027 }
1028
1029 pub fn with_daemon_provenance(
1030 mut self,
1031 pid: u32,
1032 started_at_ms: u64,
1033 executable_path: Option<PathBuf>,
1034 build_git_sha: Option<String>,
1035 build_lock_digest: Option<String>,
1036 ) -> Self {
1037 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1038 let process_start_time = process_start_time(pid);
1039 self.daemon_provenance = DaemonProvenanceFacts {
1040 build: DaemonBuildProvenance {
1041 build_git_sha,
1042 build_lock_digest,
1043 },
1044 pid: Some(pid),
1045 started_at_ms: Some(started_at_ms),
1046 start_clock: None,
1047 executable_path,
1048 executable_identity,
1049 process_start_time,
1050 probe: ExecutableIdentityProbe::default(),
1051 };
1052 self
1053 }
1054
1055 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1056 self.daemon_provenance.start_clock = Some(clock);
1057 self
1058 }
1059
1060 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1062 self.scopes
1063 .write()
1064 .unwrap_or_else(|poisoned| poisoned.into_inner())
1065 .set_wall_clock(Arc::new(clock));
1066 self
1067 }
1068
1069 #[cfg(test)]
1070 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1071 self.provenance_probe_override = Some(result);
1072 self
1073 }
1074
1075 pub fn with_capability_config(
1079 self,
1080 modules: impl IntoIterator<Item = (String, bool)>,
1081 reserved_capabilities: BTreeMap<String, String>,
1082 ) -> Self {
1083 self.capability_evaluator
1084 .configure(modules, reserved_capabilities);
1085 self
1086 }
1087
1088 pub fn with_supervisor_rescan(
1089 mut self,
1090 supervisor: Supervisor,
1091 config_path: impl Into<PathBuf>,
1092 configured_port: Option<u16>,
1093 ) -> Self {
1094 self.rescan = Some(SupervisorRescanContext {
1095 supervisor,
1096 config_path: config_path.into(),
1097 configured_port,
1098 storage_config: self.storage_config.clone(),
1099 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1100 admission_facts_targets: self.admission_facts_targets.clone(),
1101 scope_authority_owners: self.scope_authority_owners.clone(),
1102 });
1103 self
1104 }
1105
1106 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1107 self.connected_clients = connected_clients;
1108 self
1109 }
1110
1111 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1112 Arc::clone(&self.forwarding)
1113 }
1114
1115 pub(crate) fn counters(&self) -> DaemonCounters {
1116 self.counters.clone()
1117 }
1118
1119 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1122 tokio::spawn(async move {
1123 loop {
1124 self.capability_evaluator
1125 .wait_for_change_or_deadline()
1126 .await;
1127 self.refresh_capability_requirements();
1128 }
1129 });
1130 }
1131
1132 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1138 tokio::spawn(async move {
1139 loop {
1140 let has_deadlines = self
1141 .scopes
1142 .read()
1143 .unwrap_or_else(|poisoned| poisoned.into_inner())
1144 .has_deadlines();
1145 if !has_deadlines {
1146 self.scope_deadline_added.notified().await;
1147 continue;
1148 }
1149 tokio::time::sleep(Duration::from_secs(1)).await;
1150 if let Err(error) = self.sweep_expired_scopes() {
1151 warn!(%error, "scope expiry sweep failed");
1152 }
1153 }
1154 });
1155 }
1156
1157 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1158 let mut table = self
1159 .scopes
1160 .write()
1161 .unwrap_or_else(|poisoned| poisoned.into_inner());
1162 let swept = table.sweep_expired();
1163 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1164 drop(table);
1165 self.close_scope_drained_routes(drained);
1166 Ok(())
1167 }
1168
1169 fn publish_scope_changes(
1170 &self,
1171 changes: &[crate::scopes::ScopeTagChange],
1172 expired: &[crate::scopes::ScopeExpired],
1173 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1174 if changes.is_empty() {
1175 return Ok(Vec::new());
1176 }
1177 let drained = self
1178 .forwarding
1179 .publish_scope_changes(changes)
1180 .map_err(RouterError::Forwarding)?;
1181 let mut counts = HashMap::new();
1182 for route in &drained {
1183 *counts
1184 .entry((
1185 route.scope.owner.as_str(),
1186 route.scope.scope_ref.as_str(),
1187 route.scope.tag.scope_epoch,
1188 ))
1189 .or_insert(0usize) += 1;
1190 }
1191 for scope in expired {
1192 let routes_closed = counts
1193 .get(&(
1194 scope.owner.as_str(),
1195 scope.scope_ref.as_str(),
1196 scope.scope_epoch,
1197 ))
1198 .copied()
1199 .unwrap_or(0);
1200 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1201 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1202 routes_closed, "scope expired");
1203 }
1204 Ok(drained)
1205 }
1206
1207 fn runtime_capability_snapshot(
1208 &self,
1209 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1210 let runtime = self
1211 .supervisor
1212 .list()
1213 .into_iter()
1214 .map(|module| {
1215 let status = module.status().map_err(|err| {
1216 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1217 })?;
1218 Ok(RuntimeModule {
1219 module_id: status.module_id,
1220 state: status.state,
1221 enabled: status.enabled,
1222 })
1223 })
1224 .collect::<Result<Vec<_>, RouterError>>()?;
1225 let registrations = self.registry.list_registered_modules().map_err(|err| {
1231 RouterError::backend(
1232 0,
1233 0,
1234 format!("failed to list capability registrations: {err}"),
1235 )
1236 })?;
1237 let registrations = registrations
1238 .into_iter()
1239 .map(|registration| RegisteredModule {
1240 module_id: registration.manifest.module_id,
1241 module_version: registration.manifest.module_version,
1242 capabilities: registration.manifest.capabilities,
1243 })
1244 .collect();
1245 Ok((runtime, registrations))
1246 }
1247
1248 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1255 let cached_registration = RegisteredModule {
1256 module_id: registration.manifest.module_id.clone(),
1257 module_version: registration.manifest.module_version.clone(),
1258 capabilities: registration.manifest.capabilities.clone(),
1259 };
1260 if self.capability_evaluator.record_hello(&cached_registration) {
1261 warn!(
1262 module_id = %cached_registration.module_id,
1263 "capability claims drifted from the cached manifest"
1264 );
1265 }
1266 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1267 self.enforce_capability_denies();
1268 }
1269 self.refresh_capability_requirements();
1270 }
1271
1272 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1276 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1277 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1278 self.supervisor.set_swap_promotion_observer(observer);
1279 }
1280
1281 pub fn refresh_capability_requirements(&self) {
1282 match self.runtime_capability_snapshot() {
1283 Ok((runtime, registrations)) => {
1284 log_requirement_events(
1285 self.capability_evaluator
1286 .evaluate_now(&runtime, ®istrations),
1287 );
1288 }
1289 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1290 }
1291 }
1292
1293 fn enforce_capability_denies(&self) {
1297 let registrations = match self.registry.list_registered_modules() {
1298 Ok(snapshot) => snapshot,
1299 Err(err) => {
1300 warn!(error = %err, "failed to read registrations for capability deny census");
1301 return;
1302 }
1303 };
1304 let manifests = registrations
1305 .into_iter()
1306 .map(|registration| {
1307 (
1308 registration.manifest.module_id.clone(),
1309 registration.manifest,
1310 )
1311 })
1312 .collect::<BTreeMap<_, _>>();
1313 let census = match self.forwarding.route_census(None) {
1314 Ok(census) => census,
1315 Err(err) => {
1316 warn!(error = %err, "failed to read route census for capability deny enforcement");
1317 return;
1318 }
1319 };
1320
1321 for (target_module_id, routes) in census {
1322 let Some(target_manifest) = manifests.get(&target_module_id) else {
1323 continue;
1324 };
1325 let mut closed_routes = Vec::new();
1326 let mut module_goodbyes = Vec::new();
1327 for route in routes {
1328 let Principal::Reserved {
1329 module_id: opening_module_id,
1330 } = &route.principal
1331 else {
1332 continue;
1333 };
1334 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1335 continue;
1336 };
1337 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1338 continue;
1339 };
1340
1341 match self.forwarding.release_client_route(
1342 route.goodbye_target.connection_id,
1343 route.goodbye_target.channel,
1344 route.goodbye_target.epoch,
1345 ) {
1346 Ok(RouteRelease::Removed(module_goodbye)) => {
1347 warn!(
1348 opening_module_id,
1349 target_module_id,
1350 capability,
1351 "force-closing route because an attested capability deny edge now matches"
1352 );
1353 closed_routes.push(route);
1354 module_goodbyes.push(module_goodbye);
1355 }
1356 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1357 Err(err) => warn!(
1358 opening_module_id,
1359 target_module_id,
1360 capability,
1361 error = %err,
1362 "failed to force-close capability-denied route"
1363 ),
1364 }
1365 }
1366
1367 if closed_routes.is_empty() {
1368 continue;
1369 }
1370 send_route_control_pushes(
1371 &self.forwarding,
1372 closed_routes,
1373 ClientControlPush::RouteClosed {
1374 module_id: target_module_id,
1375 channels: Vec::new(),
1376 reason: RouteCloseReason::CapabilityDenied,
1377 drained: false,
1378 abandoned: 0,
1379 excluded_subscriptions: 0,
1380 terminal: Some(false),
1381 },
1382 );
1383 self.emit_route_goodbyes(module_goodbyes);
1384 }
1385 }
1386
1387 fn not_ready_reason(
1393 &self,
1394 registration: &crate::registry::ModuleRegistration,
1395 ) -> Option<NotReadyReason> {
1396 if !registration.ready {
1397 return Some(NotReadyReason {
1398 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1399 capability: None,
1400 });
1401 }
1402 self.first_unprovided_required_capability(registration)
1403 .map(|capability| NotReadyReason {
1404 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1405 capability: Some(capability),
1406 })
1407 }
1408
1409 fn first_unprovided_required_capability(
1427 &self,
1428 registration: &crate::registry::ModuleRegistration,
1429 ) -> Option<String> {
1430 let required = registration
1431 .manifest
1432 .capabilities
1433 .iter()
1434 .flat_map(|declarations| declarations.requires.iter())
1435 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1436 .map(|requirement| requirement.capability.as_str())
1437 .collect::<BTreeSet<_>>();
1438 if required.is_empty() {
1439 return None;
1440 }
1441 let module_id = registration.manifest.module_id.as_str();
1442 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1443 if required
1444 .iter()
1445 .any(|capability| verdict(capability).is_none())
1446 {
1447 self.refresh_capability_requirements();
1448 }
1449 required
1450 .into_iter()
1451 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1452 .map(str::to_string)
1453 }
1454
1455 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1456 self.capability_evaluator
1457 .statuses()
1458 .into_iter()
1459 .map(capability_requirement_status)
1460 .collect()
1461 }
1462
1463 fn deregister_connection(
1470 &self,
1471 connection_id: ConnectionId,
1472 reason: RegistrationEndReason,
1473 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1474 self.registry
1475 .deregister_connection_with_reason(connection_id, reason)
1476 }
1477
1478 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1479 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1480 return None;
1481 }
1482 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1483 parse_client_control_request(&frame.body)
1484 else {
1485 return None;
1486 };
1487 Some(target_module_id(&target).to_string())
1488 }
1489
1490 pub(crate) fn route_open_capacity_refusal(
1491 &self,
1492 ctx: &RouteCtx,
1493 frame: &Frame,
1494 target_module_id: &str,
1495 in_flight: usize,
1496 limit: usize,
1497 ) -> Result<Frame, RouterError> {
1498 self.route_open_admission_refusal_frame(
1499 ctx,
1500 frame,
1501 target_module_id,
1502 "open_admission_full",
1503 (in_flight, limit),
1504 format!(
1505 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1506 ),
1507 )
1508 }
1509
1510 fn route_open_target_capacity_refusal(
1511 &self,
1512 ctx: &RouteCtx,
1513 frame: &Frame,
1514 target_module_id: &str,
1515 in_flight: usize,
1516 ) -> Result<Frame, RouterError> {
1517 self.route_open_admission_refusal_frame(
1518 ctx,
1519 frame,
1520 target_module_id,
1521 "target_binds_full",
1522 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1523 format!(
1524 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1525 ),
1526 )
1527 }
1528
1529 fn route_open_admission_refusal_frame(
1537 &self,
1538 ctx: &RouteCtx,
1539 frame: &Frame,
1540 target_module_id: &str,
1541 reason: &'static str,
1542 (in_flight, limit): (usize, usize),
1543 message: impl Into<String>,
1544 ) -> Result<Frame, RouterError> {
1545 let code = error_codes::TARGET_UNAVAILABLE;
1546 self.counters.increment_route_open_refused(code);
1547 info!(
1548 target: "control",
1549 code,
1550 reason,
1551 module_id = ?target_module_id,
1552 connection_id = ctx.connection_id.get(),
1553 in_flight,
1554 limit,
1555 "route.open refused"
1556 );
1557 control_error_frame(frame, code, message.into())
1558 }
1559
1560 #[cfg(test)]
1566 pub fn handle_control(
1567 &self,
1568 connection_id: ConnectionId,
1569 frame: Frame,
1570 ) -> Result<Vec<Frame>, RouterError> {
1571 match frame.header.ty {
1572 FrameType::Ping => Ok(vec![pong(&frame)?]),
1573 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1574 FrameType::Goodbye => self.handle_goodbye(connection_id),
1575 ty => Ok(vec![control_error_frame(
1576 &frame,
1577 "unsupported_control_frame",
1578 format!("unsupported channel-0 frame {ty:?}"),
1579 )?]),
1580 }
1581 }
1582
1583 pub async fn handle_control_frame(
1584 &self,
1585 ctx: &RouteCtx,
1586 frame: Frame,
1587 ) -> Result<Vec<Frame>, RouterError> {
1588 self.handle_control_frame_timed(ctx, frame, None).await
1589 }
1590
1591 pub(crate) async fn handle_control_frame_timed(
1592 &self,
1593 ctx: &RouteCtx,
1594 frame: Frame,
1595 dispatch_started_at: Option<StdInstant>,
1596 ) -> Result<Vec<Frame>, RouterError> {
1597 match frame.header.ty {
1598 FrameType::Ping => Ok(vec![pong(&frame)?]),
1599 FrameType::Hello => {
1600 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1601 }
1602 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1603 FrameType::Cancel => {
1604 if self
1607 .forwarding
1608 .operator_confirms()
1609 .cancel(ctx.connection_id, frame.header.corr)
1610 || self
1611 .supervisor
1612 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1613 || self.cancel_activity_subscription(ctx, &frame)?
1614 {
1615 Ok(Vec::new())
1616 } else {
1617 Ok(vec![control_error_frame(
1618 &frame,
1619 "unknown_subscription",
1620 "no subscription has this correlation id",
1621 )?])
1622 }
1623 }
1624 FrameType::Request => {
1625 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1628 if op
1629 .as_ref()
1630 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1631 {
1632 return self.handle_activity_record(ctx, frame).await;
1633 }
1634 if op.as_ref().is_some_and(|probe| {
1635 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1636 }) {
1637 return self.handle_activity_read(ctx, frame).await;
1638 }
1639 if op
1640 .as_ref()
1641 .is_some_and(|probe| probe.op == "operator.confirm")
1642 {
1643 return self.handle_operator_confirm(ctx, frame);
1644 }
1645 if self
1646 .forwarding
1647 .module_endpoint_for_connection(ctx.connection_id)
1648 .map_err(RouterError::Forwarding)?
1649 .is_some()
1650 {
1651 if !is_known_module_request_op(&frame.body) {
1652 return Ok(vec![control_error_frame(
1653 &frame,
1654 "unsupported_control_frame",
1655 "module-originated channel-0 REQUEST is not supported",
1656 )?]);
1657 }
1658 let request = match parse_module_control_request_from_module(&frame.body) {
1659 Ok(request) => request,
1660 Err((err, ControlRequestBodyError::UnknownOp)) => {
1661 return Ok(vec![control_error_frame(
1662 &frame,
1663 "unsupported_control_frame",
1664 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1665 )?])
1666 }
1667 Err((err, ControlRequestBodyError::InvalidBody)) => {
1668 return Ok(vec![control_error_frame(
1669 &frame,
1670 "invalid_control_body",
1671 format!("malformed module control body: {err}"),
1672 )?])
1673 }
1674 };
1675 let op = module_control_request_op(&request);
1676 let corr = frame.header.corr;
1677 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1678 let result =
1679 self.handle_module_control_request(ctx.connection_id, frame, request);
1680 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1681 return result;
1682 }
1683
1684 if is_known_module_request_op(&frame.body) {
1685 return Ok(vec![control_error_frame(
1686 &frame,
1687 "not_registered",
1688 "catalog.update requires an active module registration owned by this connection",
1689 )?]);
1690 }
1691
1692 let request = match parse_client_control_request(&frame.body) {
1693 Ok(request) => request,
1694 Err((err, ControlRequestBodyError::UnknownOp)) => {
1695 return Ok(vec![control_error_frame(
1696 &frame,
1697 "unknown_control_op",
1698 format!("unknown client control op: {err}"),
1699 )?])
1700 }
1701 Err((err, ControlRequestBodyError::InvalidBody)) => {
1702 return Ok(vec![control_error_frame(
1703 &frame,
1704 "invalid_control_body",
1705 format!("malformed client control body: {err}"),
1706 )?])
1707 }
1708 };
1709 let op = client_control_request_op(&request);
1710 let corr = frame.header.corr;
1711 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1712 #[cfg(test)]
1713 if let Some(delay) = self.control_dispatch_delay {
1714 tokio::time::sleep(delay).await;
1715 }
1716 let result = self
1717 .handle_client_control_request(ctx, frame, request)
1718 .await;
1719 log_slow_control_dispatch(dispatch_started_at, op, ctx.connection_id, corr);
1720 result
1721 }
1722 FrameType::Push => {
1723 let Some(endpoint) = self
1724 .forwarding
1725 .module_endpoint_for_connection(ctx.connection_id)
1726 .map_err(RouterError::Forwarding)?
1727 else {
1728 return Ok(vec![control_error_frame(
1729 &frame,
1730 "unsupported_control_frame",
1731 "client-originated channel-0 PUSH is not supported",
1732 )?]);
1733 };
1734 self.handle_status_update(endpoint, frame)
1735 }
1736 FrameType::Response | FrameType::Error
1737 if self
1738 .forwarding
1739 .module_endpoint_for_connection(ctx.connection_id)
1740 .map_err(RouterError::Forwarding)?
1741 .is_some() =>
1742 {
1743 self.handle_module_relay_response(ctx.connection_id, frame)
1744 }
1745 ty => Ok(vec![control_error_frame(
1746 &frame,
1747 "unsupported_control_frame",
1748 format!("unsupported channel-0 frame {ty:?}"),
1749 )?]),
1750 }
1751 }
1752
1753 pub fn cleanup_connection(
1754 &self,
1755 connection_id: ConnectionId,
1756 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1757 self.cleanup_connection_with_end_reason(
1758 connection_id,
1759 RegistrationEndReason::ConnectionClosed,
1760 )
1761 }
1762
1763 fn cleanup_connection_with_end_reason(
1764 &self,
1765 connection_id: ConnectionId,
1766 requested_reason: RegistrationEndReason,
1767 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1768 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1769 self.registry
1770 .get_module_by_connection(connection_id)?
1771 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1772 .and_then(|module| module.registration_end_reason().ok().flatten())
1773 .unwrap_or(requested_reason)
1774 } else {
1775 requested_reason
1776 };
1777 let crash_closed = self
1778 .registry
1779 .get_module_by_connection(connection_id)?
1780 .and_then(|registration| {
1781 self.forwarding
1782 .module_endpoint_for_connection(connection_id)
1783 .ok()
1784 .flatten()
1785 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1786 .map(|routes| (registration.manifest.module_id, routes))
1787 });
1788 let crash_closed = crash_closed.map(|(module_id, routes)| {
1789 let terminal = match self.supervisor.get(&module_id) {
1790 None => false,
1791 Some(module) => match module.will_recover_after_connection_loss() {
1792 Ok(will_recover) => !will_recover,
1793 Err(err) => {
1794 warn!(
1795 %module_id,
1796 error = %err,
1797 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1798 );
1799 false
1800 }
1801 },
1802 };
1803 let (reason, drained) = match self
1807 .forwarding
1808 .daemon_drain_result_for_connection(connection_id)
1809 {
1810 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
1811 Ok(None) => (RouteCloseReason::Crash, false),
1812 Err(err) => {
1813 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
1814 (RouteCloseReason::Crash, false)
1815 }
1816 };
1817 (module_id, routes, reason, drained, terminal)
1818 });
1819 let registrations = self.deregister_connection(connection_id, end_reason);
1820 let cleanup = if crash_closed.is_some() {
1821 self.forwarding.cleanup_connection_counted(connection_id)
1822 } else {
1823 self.forwarding
1829 .cleanup_connection(connection_id)
1830 .map(|released| crate::forwarding::ConnectionCleanup {
1831 released,
1832 abandoned_relays: 0,
1833 })
1834 };
1835 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
1840 let abandoned = cleanup
1841 .as_ref()
1842 .map_or(0, |cleanup| cleanup.abandoned_relays);
1843 send_route_control_pushes(
1844 &self.forwarding,
1845 routes,
1846 ClientControlPush::RouteClosed {
1847 module_id,
1848 channels: Vec::new(),
1849 reason,
1850 drained,
1851 abandoned,
1852 excluded_subscriptions: 0,
1853 terminal: Some(terminal),
1854 },
1855 );
1856 }
1857 if let Ok(cleanup) = cleanup {
1858 self.emit_route_goodbyes(cleanup.released);
1859 }
1860 if matches!(®istrations, Ok(r) if !r.is_empty()) {
1864 crate::supervise::notify_registration_release();
1865 self.capability_evaluator.wake_deadline_loop();
1866 self.refresh_capability_requirements();
1867 }
1868 self.supervisor.remove_spawn_subscribers(connection_id);
1869 self.activity_streams
1870 .lock()
1871 .unwrap_or_else(|p| p.into_inner())
1872 .retain(|(id, _), task| {
1873 if *id == connection_id {
1874 task.abort();
1875 false
1876 } else {
1877 true
1878 }
1879 });
1880 self.hello_launch_nonces
1883 .lock()
1884 .unwrap_or_else(|poisoned| poisoned.into_inner())
1885 .forget(connection_id);
1886 self.scopes
1887 .write()
1888 .unwrap_or_else(|poisoned| poisoned.into_inner())
1889 .release_connection(connection_id);
1890 registrations
1891 }
1892
1893 pub(crate) fn handle_route_goodbye(
1894 &self,
1895 connection_id: ConnectionId,
1896 route_channel: u16,
1897 route_epoch: u32,
1898 ) -> Result<bool, RouterError> {
1899 debug!(
1900 connection_id = connection_id.get(),
1901 route_channel, route_epoch, "handling route GOODBYE"
1902 );
1903 let RouteRelease::Removed(released_route) = self
1904 .forwarding
1905 .release_client_route(connection_id, route_channel, route_epoch)
1906 .map_err(RouterError::Forwarding)?
1907 else {
1908 return Ok(false);
1909 };
1910 self.emit_route_goodbyes(vec![released_route]);
1911 Ok(true)
1912 }
1913
1914 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
1915 for released in released_routes {
1916 let frame = match Frame::build_with_version(
1917 released.negotiated_ver,
1918 FrameType::Goodbye,
1919 control_flags(),
1920 released.channel,
1921 released.epoch,
1922 0,
1923 Vec::new(),
1924 ) {
1925 Ok(frame) => frame,
1926 Err(err) => {
1927 warn!(
1928 route_channel = released.channel,
1929 error = %err,
1930 "failed to build route GOODBYE frame"
1931 );
1932 continue;
1933 }
1934 };
1935 if !released.close_on_delivery_failure() {
1936 crate::forwarding::send_module_route_goodbye(
1937 &self.counters,
1938 &released.sink,
1939 frame,
1940 released.module_id.as_deref(),
1941 "client route released",
1942 );
1943 continue;
1944 }
1945 if let Err(err) = released.sink.try_send(frame) {
1946 warn!(
1947 target_connection_id = released.connection_id.get(),
1948 route_channel = released.channel,
1949 error = %err,
1950 "route GOODBYE was not delivered to client; closing target connection"
1951 );
1952 if self
1953 .forwarding
1954 .escalate_client_delivery_failure(
1955 released.connection_id,
1956 released.channel,
1957 released.epoch,
1958 CloseReason::new(
1959 "route_goodbye_delivery_failed",
1960 format!(
1961 "failed to enqueue route GOODBYE for channel {}: {err}",
1962 released.channel
1963 ),
1964 ),
1965 crate::forwarding::UndeliveredFrame {
1966 module_id: released.module_id.as_deref(),
1967 sink: &released.sink,
1968 },
1969 )
1970 .unwrap_or(false)
1971 {
1972 self.counters.increment_goodbye_relay_client_failed();
1973 }
1974 }
1975 }
1976 }
1977
1978 fn send_abandoned_route_bind_goodbye(
1990 &self,
1991 module_sink: &crate::FrameSink,
1992 negotiated_ver: u8,
1993 module_channel: u16,
1994 module_epoch: u32,
1995 ) {
1996 let frame = match Frame::build_with_version(
1997 negotiated_ver,
1998 FrameType::Goodbye,
1999 control_flags(),
2000 module_channel,
2001 module_epoch,
2002 0,
2003 Vec::new(),
2004 ) {
2005 Ok(frame) => frame,
2006 Err(err) => {
2007 warn!(
2008 route_channel = module_channel,
2009 error = %err,
2010 "failed to build GOODBYE for abandoned route.bind"
2011 );
2012 return;
2013 }
2014 };
2015 crate::forwarding::send_module_route_goodbye(
2016 &self.counters,
2017 module_sink,
2018 frame,
2019 None,
2020 "abandoned route.bind",
2021 );
2022 }
2023
2024 fn handle_hello(
2025 &self,
2026 connection_id: ConnectionId,
2027 sink: Option<crate::FrameSink>,
2028 frame: Frame,
2029 ) -> Result<Vec<Frame>, RouterError> {
2030 debug!(
2031 connection_id = connection_id.get(),
2032 corr = frame.header.corr,
2033 "handling HELLO"
2034 );
2035 if self
2039 .registry
2040 .get_module_by_connection(connection_id)
2041 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2042 .is_some()
2043 {
2044 return Ok(vec![control_error_frame(
2045 &frame,
2046 "invalid_hello",
2047 "connection is already registered as a module",
2048 )?]);
2049 }
2050 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2051 Ok(value) => value,
2052 Err(err) => {
2053 return Ok(vec![control_error_frame(
2054 &frame,
2055 "invalid_hello",
2056 format!("malformed HELLO body: {err}"),
2057 )?])
2058 }
2059 };
2060 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2061 return Ok(vec![control_error_frame(
2062 &frame,
2063 "invalid_capability_grammar",
2064 err.to_string(),
2065 )?]);
2066 }
2067 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2068 return Ok(vec![control_error_frame(
2069 &frame,
2070 "invalid_manifest",
2071 err.to_string(),
2072 )?]);
2073 }
2074 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2075 return Ok(vec![control_error_frame(
2076 &frame,
2077 "invalid_event_declaration",
2078 err.to_string(),
2079 )?]);
2080 }
2081 if let Some(provenance) = hello_value
2082 .get("manifest")
2083 .and_then(|manifest| manifest.get("provenance"))
2084 {
2085 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2086 return Ok(vec![control_error_frame(
2087 &frame,
2088 "invalid_manifest",
2089 format!("malformed manifest provenance: {err}"),
2090 )?]);
2091 }
2092 }
2093 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2094 Ok(hello) => hello,
2095 Err(err) => {
2096 return Ok(vec![control_error_frame(
2097 &frame,
2098 "invalid_hello",
2099 format!("malformed HELLO body: {err}"),
2100 )?])
2101 }
2102 };
2103
2104 if hello.protocol_ver != hello.manifest.protocol_ver {
2105 return Ok(vec![control_error_frame(
2106 &frame,
2107 "invalid_manifest",
2108 format!(
2109 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2110 hello.protocol_ver, hello.manifest.protocol_ver
2111 ),
2112 )?]);
2113 }
2114
2115 if hello.manifest.module_id.trim().is_empty() {
2116 return Ok(vec![control_error_frame(
2117 &frame,
2118 "invalid_manifest",
2119 "manifest module_id must not be empty",
2120 )?]);
2121 }
2122
2123 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2124 Ok(negotiated_ver) => negotiated_ver,
2125 Err(message) => {
2126 return Ok(vec![control_error_frame(
2127 &frame,
2128 "version_unsupported",
2129 message,
2130 )?])
2131 }
2132 };
2133
2134 let swap_admission = self
2142 .supervisor
2143 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2144 if swap_admission == SwapHelloAdmission::Refused {
2145 warn!(
2146 module_id = %hello.manifest.module_id,
2147 connection_id = connection_id.get(),
2148 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2149 );
2150 return Ok(vec![control_error_frame(
2151 &frame,
2152 "swap_token_invalid",
2153 format!(
2154 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2155 hello.manifest.module_id
2156 ),
2157 )?]);
2158 }
2159 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2160
2161 if let Some(rejection) = (!swap_candidate)
2169 .then(|| {
2170 self.supervisor.reserved_hello_rejection(
2171 &hello.manifest.module_id,
2172 hello.launch_nonce.as_deref(),
2173 )
2174 })
2175 .flatten()
2176 {
2177 let message = match rejection {
2178 ReservedHelloRejection::Exact { module_id } => format!(
2179 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2180 ),
2181 ReservedHelloRejection::Prefix {
2182 prefix,
2183 owner_module_id,
2184 } => format!(
2185 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2186 hello.manifest.module_id
2187 ),
2188 };
2189 return Ok(vec![control_error_frame(
2190 &frame,
2191 "reserved_module",
2192 message,
2193 )?]);
2194 }
2195
2196 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2197 &hello.manifest.module_id,
2198 hello.manifest.capabilities.as_ref(),
2199 );
2200 if let Some(refusal) = reserved_capability_refusals.first() {
2201 let capability = refusal.capability.clone();
2202 let bound_module = refusal.claimants[0].clone();
2203 log_duplicate_claim_events(reserved_capability_refusals);
2204 return Ok(vec![control_error_frame(
2205 &frame,
2206 "reserved_capability",
2207 format!(
2208 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2209 capability, bound_module, hello.manifest.module_id
2210 ),
2211 )?]);
2212 }
2213
2214 if self
2217 .forwarding
2218 .connection_has_client_routes(connection_id)
2219 .map_err(RouterError::Forwarding)?
2220 {
2221 return Ok(vec![control_error_frame(
2222 &frame,
2223 "invalid_hello",
2224 "connection has open client routes and cannot also register as a module",
2225 )?]);
2226 }
2227
2228 self.hello_launch_nonces
2233 .lock()
2234 .unwrap_or_else(|poisoned| poisoned.into_inner())
2235 .record(connection_id, hello.launch_nonce.as_deref());
2236 let control_ops = effective_module_control_ops(hello.control_ops);
2237 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2240 if swap_candidate {
2241 return self.register_swap_candidate(
2242 connection_id,
2243 sink,
2244 &frame,
2245 hello.manifest,
2246 negotiated_ver,
2247 control_ops,
2248 hello_ack,
2249 );
2250 }
2251 let registration = match self.registry.register_unpublished_with_control_ops(
2258 hello.manifest,
2259 negotiated_ver,
2260 connection_id,
2261 control_ops,
2262 ) {
2263 Ok(registration) => registration,
2264 Err(RegistryError::DuplicateModuleId { module_id }) => {
2265 return Ok(vec![control_error_frame(
2266 &frame,
2267 "duplicate_module_id",
2268 format!(
2269 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2270 ),
2271 )?])
2272 }
2273 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2274 return Ok(vec![control_error_frame(
2275 &frame,
2276 "invalid_module_id",
2277 err.to_string(),
2278 )?])
2279 }
2280 Err(err) => {
2281 return Ok(vec![control_error_frame(
2282 &frame,
2283 "registry_error",
2284 err.to_string(),
2285 )?])
2286 }
2287 };
2288 #[cfg(test)]
2289 self.hold_between_registry_and_forwarding();
2290
2291 let reply = if let Some(sink) = sink {
2292 let concurrency = manifest_concurrency(®istration.manifest);
2308 if let Err(err) = self.forwarding.register_module_connection_acked(
2309 connection_id,
2310 registration.manifest.module_id.clone(),
2311 negotiated_ver,
2312 concurrency,
2313 sink,
2314 hello_ack,
2315 ) {
2316 if matches!(
2321 self.deregister_connection(
2322 connection_id,
2323 RegistrationEndReason::RegistrationFailed,
2324 ),
2325 Ok(r) if !r.is_empty()
2326 ) {
2327 crate::supervise::notify_registration_release();
2328 }
2329 return Ok(vec![control_error_frame(
2330 &frame,
2331 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2332 "invalid_hello"
2333 } else {
2334 forwarding_error_code(&err)
2335 },
2336 err.to_string(),
2337 )?]);
2338 }
2339 Vec::new()
2340 } else {
2341 vec![hello_ack]
2344 };
2345
2346 if !self
2351 .registry
2352 .publish_registration(connection_id)
2353 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2354 {
2355 debug!(
2356 module_id = %registration.manifest.module_id,
2357 connection_id = connection_id.get(),
2358 "registration was removed before it could be published"
2359 );
2360 }
2361
2362 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2371 info!(
2372 module_id = %registration.manifest.module_id,
2373 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2374 );
2375 }
2376
2377 self.apply_registration_capabilities(®istration);
2378
2379 self.supervisor
2380 .record_activity_registration(®istration.manifest.module_id);
2381
2382 info!(
2383 module_id = %registration.manifest.module_id,
2384 module_version = %registration.manifest.module_version,
2385 negotiated_ver,
2386 routable_provider = manifest_provides_routable_role(®istration.manifest),
2387 connection_id = connection_id.get(),
2388 "module registered"
2389 );
2390
2391 Ok(reply)
2392 }
2393
2394 #[allow(clippy::too_many_arguments)]
2411 fn register_swap_candidate(
2412 &self,
2413 connection_id: ConnectionId,
2414 sink: Option<crate::FrameSink>,
2415 frame: &Frame,
2416 manifest: ModuleManifest,
2417 negotiated_ver: u8,
2418 control_ops: Vec<String>,
2419 hello_ack: Frame,
2420 ) -> Result<Vec<Frame>, RouterError> {
2421 let module_id = manifest.module_id.clone();
2422 let registration = match self.registry.register_unpublished_candidate_with_control_ops(
2423 manifest,
2424 negotiated_ver,
2425 connection_id,
2426 control_ops,
2427 ) {
2428 Ok(registration) => registration,
2429 Err(RegistryError::DuplicateModuleId { module_id }) => {
2430 return Ok(vec![control_error_frame(
2431 frame,
2432 "duplicate_module_id",
2433 format!(
2434 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2435 ),
2436 )?])
2437 }
2438 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2439 return Ok(vec![control_error_frame(
2440 frame,
2441 "invalid_module_id",
2442 err.to_string(),
2443 )?])
2444 }
2445 Err(err) => {
2446 return Ok(vec![control_error_frame(
2447 frame,
2448 "registry_error",
2449 err.to_string(),
2450 )?])
2451 }
2452 };
2453 #[cfg(test)]
2454 self.hold_between_registry_and_forwarding();
2455 let reply = if let Some(sink) = sink {
2456 let concurrency = manifest_concurrency(®istration.manifest);
2460 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2461 connection_id,
2462 module_id.clone(),
2463 negotiated_ver,
2464 concurrency,
2465 sink,
2466 hello_ack,
2467 ) {
2468 if matches!(
2469 self.deregister_connection(
2470 connection_id,
2471 RegistrationEndReason::RegistrationFailed,
2472 ),
2473 Ok(r) if !r.is_empty()
2474 ) {
2475 crate::supervise::notify_registration_release();
2476 }
2477 return Ok(vec![control_error_frame(
2478 frame,
2479 forwarding_error_code(&err),
2480 err.to_string(),
2481 )?]);
2482 }
2483 Vec::new()
2484 } else {
2485 vec![hello_ack]
2486 };
2487 if !self
2488 .registry
2489 .publish_registration(connection_id)
2490 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2491 {
2492 debug!(
2493 module_id = %module_id,
2494 connection_id = connection_id.get(),
2495 "swap candidate registration was removed before it could be published"
2496 );
2497 }
2498 self.supervisor.mark_swap_candidate_admitted(&module_id);
2499 info!(
2500 module_id = %module_id,
2501 module_version = %registration.manifest.module_version,
2502 negotiated_ver,
2503 ready = registration.ready,
2504 connection_id = connection_id.get(),
2505 "swap candidate registered; not routable until cutover"
2506 );
2507 Ok(reply)
2508 }
2509
2510 fn build_hello_ack(
2511 &self,
2512 frame: &Frame,
2513 negotiated_ver: u8,
2514 module_id: &str,
2515 ) -> Result<Frame, RouterError> {
2516 let ack = ModuleHelloAckBody {
2517 negotiated_ver,
2518 subc_ops: module_subc_ops(),
2519 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2520 storage: self
2521 .storage_config
2522 .as_ref()
2523 .map(|cfg| cfg.descriptor_for(module_id)),
2524 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2525 };
2526 let body = serde_json::to_vec(&ack).map_err(|err| {
2527 RouterError::backend(
2528 0,
2529 frame.header.corr,
2530 format!("failed to encode HELLO_ACK: {err}"),
2531 )
2532 })?;
2533
2534 Frame::build_with_version(
2535 negotiated_ver,
2536 FrameType::HelloAck,
2537 control_flags(),
2538 0,
2539 0,
2540 frame.header.corr,
2541 body,
2542 )
2543 .map_err(RouterError::FrameBuild)
2544 }
2545
2546 async fn handle_client_control_request(
2547 &self,
2548 ctx: &RouteCtx,
2549 frame: Frame,
2550 request: ClientControlRequest,
2551 ) -> Result<Vec<Frame>, RouterError> {
2552 match request {
2553 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2554 ClientControlRequest::CatalogList { module_id } => {
2555 self.handle_catalog_list(frame, module_id)
2556 }
2557 ClientControlRequest::RouteOpen {
2558 target,
2559 identity,
2560 consumer_identity,
2561 consumer_capabilities,
2562 role_versions,
2563 admission_facts,
2564 scope,
2565 } => {
2566 self.handle_route_open(
2567 ctx,
2568 frame,
2569 RouteOpenRequest {
2570 target,
2571 identity,
2572 consumer_identity,
2573 consumer_capabilities,
2574 role_versions,
2575 admission_facts,
2576 scope,
2577 },
2578 )
2579 .await
2580 }
2581 ClientControlRequest::RoutePoll {
2582 route_channel,
2583 route_epoch,
2584 kind,
2585 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2586 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2587 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2588 self.handle_supervisor_spawn_snapshot(frame)
2589 }
2590 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2591 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2592 }
2593 ClientControlRequest::SupervisorRestart {
2594 module_id,
2595 drain_timeout_ms,
2596 } => {
2597 self.log_supervisor_request_received(
2598 ctx,
2599 frame.header.corr,
2600 ops::SUPERVISOR_RESTART,
2601 Some(&module_id),
2602 None,
2603 )?;
2604 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2605 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2606 .await
2607 }
2608 ClientControlRequest::SupervisorSwap {
2609 module_id,
2610 ready_timeout_ms,
2611 } => {
2612 self.log_supervisor_request_received(
2613 ctx,
2614 frame.header.corr,
2615 ops::SUPERVISOR_SWAP,
2616 Some(&module_id),
2617 None,
2618 )?;
2619 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2620 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2621 .await
2622 }
2623 ClientControlRequest::SupervisorReload { module_id } => {
2624 self.log_supervisor_request_received(
2625 ctx,
2626 frame.header.corr,
2627 ops::SUPERVISOR_RELOAD,
2628 Some(&module_id),
2629 None,
2630 )?;
2631 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2632 self.handle_supervisor_reload(frame, module_id, requester)
2633 .await
2634 }
2635 ClientControlRequest::SupervisorRescan { preview } => {
2636 if !preview {
2637 self.log_supervisor_request_received(
2638 ctx,
2639 frame.header.corr,
2640 ops::SUPERVISOR_RESCAN,
2641 None,
2642 None,
2643 )?;
2644 }
2645 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2646 self.handle_supervisor_rescan(frame, preview, requester)
2647 .await
2648 }
2649 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2650 self.log_supervisor_request_received(
2651 ctx,
2652 frame.header.corr,
2653 ops::SUPERVISOR_RELEASE_RESERVED,
2654 Some(&module_id),
2655 None,
2656 )?;
2657 self.handle_supervisor_release_reserved(frame, module_id)
2658 .await
2659 }
2660 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2661 self.log_supervisor_request_received(
2662 ctx,
2663 frame.header.corr,
2664 ops::SUPERVISOR_SET_ENABLED,
2665 Some(&module_id),
2666 Some(enabled),
2667 )?;
2668 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2669 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2670 .await
2671 }
2672 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2673 self.handle_supervisor_health_probe(frame, module_id).await
2674 }
2675 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2676 ClientControlRequest::SupervisorRoutes { module_id } => {
2677 self.handle_supervisor_routes(frame, module_id)
2678 }
2679 ClientControlRequest::SupervisorProvenance { module_id } => {
2680 self.handle_supervisor_provenance(frame, module_id).await
2681 }
2682 ClientControlRequest::SupervisorStderrTail {
2683 module_id,
2684 max_lines,
2685 max_bytes,
2686 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2687 ClientControlRequest::SupervisorTerminals { module_id } => {
2688 self.handle_supervisor_terminals(frame, module_id).await
2689 }
2690 }
2691 }
2692
2693 fn log_supervisor_request_received(
2694 &self,
2695 ctx: &RouteCtx,
2696 corr: u64,
2697 op: &'static str,
2698 module_id: Option<&str>,
2699 enabled: Option<bool>,
2700 ) -> Result<(), RouterError> {
2701 let caller = self.supervisor_requester(ctx, corr)?;
2702 let caller = principal_label(&caller);
2703 info!(
2706 target: "control",
2707 op,
2708 module_id,
2709 enabled,
2710 connection_id = ctx.connection_id.get(),
2711 caller = %caller,
2712 "supervisor request received"
2713 );
2714 Ok(())
2715 }
2716
2717 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2718 Ok(self
2719 .registry
2720 .get_module_by_connection(ctx.connection_id)
2721 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2722 .map(|registration| Principal::Reserved {
2723 module_id: registration.manifest.module_id,
2724 })
2725 .unwrap_or(Principal::Direct))
2726 }
2727
2728 fn handle_module_control_request(
2729 &self,
2730 connection_id: ConnectionId,
2731 frame: Frame,
2732 request: ModuleControlRequestFromModule,
2733 ) -> Result<Vec<Frame>, RouterError> {
2734 match request {
2735 ModuleControlRequestFromModule::CatalogUpdate {
2736 provides,
2737 capabilities,
2738 ready,
2739 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2740 ModuleControlRequestFromModule::LiveRoots {} => {
2741 let registered = self
2742 .registry
2743 .get_module_by_connection(connection_id)
2744 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2745 let Some(registration) = registered else {
2746 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2747 };
2748 let response = self
2749 .forwarding
2750 .live_roots(®istration.manifest.module_id)
2751 .map_err(RouterError::Forwarding)?;
2752 Ok(vec![control_response_body_frame(
2753 &frame,
2754 &response,
2755 "ModuleControlResponseToModule::LiveRoots",
2756 )?])
2757 }
2758 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2759 self.handle_scope_sync(connection_id, frame, generation, scopes)
2760 }
2761 ModuleControlRequestFromModule::ScopeApply {
2762 generation,
2763 upsert,
2764 end,
2765 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2766 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2767 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2768 }
2769 }
2770 }
2771
2772 fn handle_operator_confirm(
2773 &self,
2774 ctx: &RouteCtx,
2775 frame: Frame,
2776 ) -> Result<Vec<Frame>, RouterError> {
2777 use crate::operator_confirm::{audit, Outcome};
2778 let registration = self
2779 .registry
2780 .get_module_by_connection(ctx.connection_id)
2781 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2782 let Some(registration) = registration else {
2783 let outcome = Outcome::refusal("not_registered");
2784 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2785 return Ok(vec![outcome.frame(&frame)]);
2786 };
2787 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
2788 Ok(request) => request,
2789 Err(_) => {
2790 let outcome = Outcome::refusal("invalid_control_body");
2791 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
2792 return Ok(vec![outcome.frame(&frame)]);
2793 }
2794 };
2795 let module_id = registration.manifest.module_id;
2796 let nonce = self
2800 .hello_launch_nonces
2801 .lock()
2802 .unwrap_or_else(|p| p.into_inner())
2803 .nonce(ctx.connection_id)
2804 .map(str::to_owned);
2805 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
2806 self.supervisor
2807 .spawned_consumer_authorized(&module_id, nonce)
2808 });
2809 let confirms = self.forwarding.operator_confirms();
2810 self.forwarding
2811 .with_operator_route(
2812 ctx.connection_id,
2813 request.route_channel,
2814 request.route_epoch,
2815 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
2816 )
2817 .map_err(RouterError::Forwarding)
2818 }
2819
2820 fn handle_scope_sync(
2824 &self,
2825 connection_id: ConnectionId,
2826 frame: Frame,
2827 generation: u64,
2828 scopes: Vec<ScopeRecord>,
2829 ) -> Result<Vec<Frame>, RouterError> {
2830 self.handle_scope_change(connection_id, frame, generation, scopes, None)
2831 }
2832
2833 fn handle_scope_change(
2834 &self,
2835 connection_id: ConnectionId,
2836 frame: Frame,
2837 generation: u64,
2838 scopes: Vec<ScopeRecord>,
2839 end: Option<Vec<ScopeEnd>>,
2840 ) -> Result<Vec<Frame>, RouterError> {
2841 let is_apply = end.is_some();
2842 let Some(registration) = self
2843 .registry
2844 .get_module_by_connection(connection_id)
2845 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2846 else {
2847 return Ok(vec![control_error_frame(
2848 &frame,
2849 "not_registered",
2850 "scope.sync and scope.apply require an active module registration owned by this connection",
2851 )?]);
2852 };
2853 let owner = registration.manifest.module_id;
2854 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
2855 let is_current_launch = |connection: ConnectionId| {
2856 self.hello_launch_nonces
2857 .lock()
2858 .unwrap_or_else(|poisoned| poisoned.into_inner())
2859 .presented(connection, current_nonce.as_deref())
2860 };
2861 let mut table = self
2866 .scopes
2867 .write()
2868 .unwrap_or_else(|poisoned| poisoned.into_inner());
2869 let outcome = match end {
2870 Some(end) => table.apply(
2871 &owner,
2872 connection_id,
2873 is_current_launch,
2874 generation,
2875 scopes,
2876 end,
2877 ),
2878 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
2879 };
2880 let drained = match &outcome {
2881 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
2882 Err(_) => Vec::new(),
2883 };
2884 let has_deadlines = outcome.is_ok() && table.has_deadlines();
2885 drop(table);
2886 if has_deadlines {
2887 self.scope_deadline_added.notify_one();
2888 }
2889 match outcome {
2890 Ok(applied) => {
2891 let counts = ScopeOutcomeCounts::of(&applied.results);
2892 info!(
2893 owner = %owner,
2894 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2895 generation,
2896 records = applied.results.len(),
2897 created = counts.created,
2898 replaced = counts.replaced,
2899 updated = counts.updated,
2900 unchanged = counts.unchanged,
2901 refused = counts.refused,
2902 ended = applied.ended.len(),
2903 tag_changes = applied.tag_changes.len(),
2904 routes_closed = drained.len(),
2905 "scope change accepted"
2906 );
2907 for refused in applied
2913 .results
2914 .iter()
2915 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
2916 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
2917 {
2918 warn!(
2919 owner = %owner,
2920 generation,
2921 scope_ref = %refused.scope_ref,
2922 scope_epoch = refused.scope_epoch,
2923 code = refused.code.as_deref().unwrap_or(""),
2924 "scope record refused"
2925 );
2926 }
2927 self.close_scope_drained_routes(drained);
2928 let response = if is_apply {
2929 ModuleControlResponseToModule::ScopeApply {
2930 generation,
2931 results: applied.results,
2932 end_results: applied.end_results,
2933 ended: applied.ended,
2934 }
2935 } else {
2936 ModuleControlResponseToModule::ScopeSync {
2937 generation,
2938 results: applied.results,
2939 ended: applied.ended,
2940 }
2941 };
2942 Ok(vec![control_response_body_frame(
2943 &frame,
2944 &response,
2945 "scope change response",
2946 )?])
2947 }
2948 Err(refusal) => {
2949 info!(
2950 owner = %owner,
2951 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
2952 generation,
2953 code = refusal.code,
2954 "scope change refused"
2955 );
2956 Ok(vec![control_error_frame(
2957 &frame,
2958 refusal.code,
2959 refusal.message,
2960 )?])
2961 }
2962 }
2963 }
2964
2965 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
2971 if drained.is_empty() {
2972 return;
2973 }
2974 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
2975 BTreeMap::new();
2976 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
2977 for route in drained {
2978 warn!(
2979 module_id = %route.module_id,
2980 reason = ?route.reason,
2981 client_connection_id = route.client.connection_id.get(),
2982 route_channel = route.client.channel,
2983 "closing route because its scope changed"
2984 );
2985 pushes
2986 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
2987 .or_insert_with(|| (route.reason, Vec::new()))
2988 .1
2989 .push(EndpointRoute {
2990 goodbye_target: route.client.clone(),
2991 principal: Principal::Unverified,
2992 bound_at: Instant::now(),
2993 draining: false,
2994 drain_reason: None,
2995 });
2996 goodbyes.push(route.module);
2997 goodbyes.push(route.client);
2998 }
2999 for ((module_id, _), (reason, routes)) in pushes {
3000 send_route_control_pushes(
3001 &self.forwarding,
3002 routes,
3003 ClientControlPush::RouteClosed {
3004 module_id,
3005 channels: Vec::new(),
3006 reason,
3007 drained: false,
3008 abandoned: 0,
3009 excluded_subscriptions: 0,
3010 terminal: Some(false),
3011 },
3012 );
3013 }
3014 self.emit_route_goodbyes(goodbyes);
3015 }
3016
3017 fn handle_scope_describe(
3020 &self,
3021 connection_id: ConnectionId,
3022 frame: Frame,
3023 owner: Principal,
3024 scope_ref: String,
3025 ) -> Result<Vec<Frame>, RouterError> {
3026 let registered = self
3027 .registry
3028 .get_module_by_connection(connection_id)
3029 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3030 if registered.is_none() {
3031 return Ok(vec![control_error_frame(
3032 &frame,
3033 "not_registered",
3034 "scope.describe requires an active module registration owned by this connection",
3035 )?]);
3036 }
3037 let description = self
3038 .scopes
3039 .read()
3040 .unwrap_or_else(|poisoned| poisoned.into_inner())
3041 .describe(&owner, &scope_ref);
3042 let owner_configured = match &owner {
3043 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3048 _ => false,
3049 };
3050 let response = ModuleControlResponseToModule::ScopeDescribe {
3051 status: description.status,
3052 scope_epoch: description.scope_epoch,
3053 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3054 owner_synced: description.owner_synced,
3055 owner_configured,
3056 scope: description.stamp,
3057 };
3058 Ok(vec![control_response_body_frame(
3059 &frame,
3060 &response,
3061 "ModuleControlResponseToModule::ScopeDescribe",
3062 )?])
3063 }
3064
3065 fn handle_catalog_update(
3066 &self,
3067 connection_id: ConnectionId,
3068 frame: Frame,
3069 provides: Vec<ProviderRole>,
3070 capabilities: Option<CapabilityDeclarations>,
3071 ready: Option<bool>,
3072 ) -> Result<Vec<Frame>, RouterError> {
3073 self.refresh_capability_requirements();
3074 let Some(registration) = self
3075 .registry
3076 .get_module_by_connection(connection_id)
3077 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3078 else {
3079 return Ok(vec![control_error_frame(
3080 &frame,
3081 "not_registered",
3082 "catalog.update requires an active module registration owned by this connection",
3083 )?]);
3084 };
3085
3086 if let Some(message) =
3087 catalog_update_frozen_field_message(®istration.manifest, &provides)
3088 {
3089 return Ok(vec![control_error_frame(
3090 &frame,
3091 "catalog_update_frozen_field",
3092 message,
3093 )?]);
3094 }
3095
3096 let mut candidate = registration.manifest.clone();
3097 candidate.provides = provides.clone();
3098 candidate.capabilities = capabilities
3099 .clone()
3100 .or_else(|| registration.manifest.capabilities.clone());
3101 if let Err(err) = candidate.validate_capability_grammar() {
3102 return Ok(vec![control_error_frame(
3103 &frame,
3104 "invalid_capability_grammar",
3105 err.to_string(),
3106 )?]);
3107 }
3108
3109 let mut conflicts = self
3112 .capability_evaluator
3113 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3114 if let Some(conflict) = conflicts.first() {
3115 let message = format!(
3116 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3117 conflict.capability, conflict.claimants[0], candidate.module_id
3118 );
3119 for conflict in &mut conflicts {
3120 conflict.source = DuplicateClaimSource::CatalogUpdate;
3121 }
3122 log_duplicate_claim_events(conflicts);
3123 return Ok(vec![control_error_frame(
3124 &frame,
3125 "reserved_capability",
3126 message,
3127 )?]);
3128 }
3129
3130 let updated = self
3131 .registry
3132 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3133 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3134 if updated.is_none() {
3135 return Ok(vec![control_error_frame(
3136 &frame,
3137 "not_registered",
3138 "catalog.update requires an active module registration owned by this connection",
3139 )?]);
3140 }
3141 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3142 log_duplicate_claim_events(
3143 self.capability_evaluator
3144 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3145 );
3146 }
3147 if capability_census_trigger(
3148 registration.manifest.capabilities.as_ref(),
3149 updated
3150 .as_ref()
3151 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3152 ) {
3153 self.enforce_capability_denies();
3154 }
3155 self.refresh_capability_requirements();
3156
3157 let response = ModuleControlResponseToModule::CatalogUpdate {};
3158 control_response_body_frame(
3159 &frame,
3160 &response,
3161 "ModuleControlResponseToModule::CatalogUpdate",
3162 )
3163 .map(|frame| vec![frame])
3164 }
3165
3166 pub(crate) fn with_activity_ledger_config(
3167 mut self,
3168 run_dir: &Path,
3169 incarnation: String,
3170 config: crate::activity::ActivityConfig,
3171 ) -> Self {
3172 self.activity = Some(crate::activity::ActivityLedger::open_config(
3173 run_dir.join("activity.jsonl"),
3174 incarnation,
3175 config,
3176 ));
3177 self.supervisor.set_activity(self.activity.clone());
3178 self
3179 }
3180
3181 pub(crate) async fn configure_activity(
3182 &self,
3183 config: crate::activity::ActivityConfig,
3184 ) -> Result<(), &'static str> {
3185 match &self.activity {
3186 Some(ledger) => ledger.configure(config).await,
3187 None => Ok(()),
3188 }
3189 }
3190
3191 fn cancel_activity_subscription(
3192 &self,
3193 ctx: &RouteCtx,
3194 frame: &Frame,
3195 ) -> Result<bool, RouterError> {
3196 let task = self
3197 .activity_streams
3198 .lock()
3199 .unwrap_or_else(|p| p.into_inner())
3200 .remove(&(ctx.connection_id, frame.header.corr));
3201 let Some(task) = task else {
3202 return Ok(false);
3203 };
3204 task.abort();
3205 let end = Frame::build_with_version(
3206 response_version(frame),
3207 FrameType::StreamEnd,
3208 control_flags(),
3209 0,
3210 0,
3211 frame.header.corr,
3212 Vec::new(),
3213 )
3214 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3215 let sink = ctx.egress.clone();
3216 tokio::spawn(async move {
3217 let _ = sink.send(end).await;
3218 });
3219 Ok(true)
3220 }
3221
3222 async fn handle_activity_read(
3223 &self,
3224 ctx: &RouteCtx,
3225 frame: Frame,
3226 ) -> Result<Vec<Frame>, RouterError> {
3227 use subc_control::{
3228 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3229 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3230 };
3231 #[derive(Deserialize)]
3232 struct Identity {
3233 #[serde(default)]
3234 consumer_identity: Option<ConsumerIdentity>,
3235 }
3236 let principal = match self
3240 .forwarding
3241 .module_id_for_connection(ctx.connection_id)
3242 .map_err(RouterError::Forwarding)?
3243 {
3244 Some(module_id) => Ok(Principal::Reserved { module_id }),
3245 None => {
3246 let identity: Identity = match serde_json::from_slice(&frame.body) {
3247 Ok(identity) => identity,
3248 Err(_) => {
3249 return Ok(vec![control_error_frame(
3250 &frame,
3251 ACTIVITY_MALFORMED,
3252 "malformed activity read",
3253 )?])
3254 }
3255 };
3256 self.route_open_principal(&frame, identity.consumer_identity)?
3257 }
3258 };
3259 let admitted = matches!(principal, Ok(Principal::Direct))
3260 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3261 if !admitted {
3262 return Ok(vec![control_error_frame(
3263 &frame,
3264 ACTIVITY_READ_FORBIDDEN,
3265 "activity reads require an operator or trusted mirror principal",
3266 )?]);
3267 }
3268 let Some(ledger) = &self.activity else {
3269 return Ok(vec![control_error_frame(
3270 &frame,
3271 ACTIVITY_UNAVAILABLE,
3272 "activity ledger unavailable",
3273 )?]);
3274 };
3275 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3276 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3277 {
3278 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3279 Ok(request) => ledger.query(request).await,
3280 Err(_) => Err(ACTIVITY_MALFORMED),
3281 };
3282 return match result {
3283 Ok(reply) => Ok(vec![control_response_body_frame(
3284 &frame,
3285 &reply,
3286 ops::ACTIVITY_QUERY,
3287 )?]),
3288 Err(code) => Ok(vec![control_error_frame(
3289 &frame,
3290 code,
3291 "activity query refused",
3292 )?]),
3293 };
3294 }
3295 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3296 Ok(request) => request,
3297 Err(_) => {
3298 return Ok(vec![control_error_frame(
3299 &frame,
3300 ACTIVITY_MALFORMED,
3301 "malformed activity subscription",
3302 )?])
3303 }
3304 };
3305 let mut subscription = match ledger.subscribe(request.since).await {
3306 Ok(subscription) => subscription,
3307 Err(code) => {
3308 return Ok(vec![control_error_frame(
3309 &frame,
3310 code,
3311 "activity subscription refused",
3312 )?])
3313 }
3314 };
3315 let key = (ctx.connection_id, frame.header.corr);
3316 let sink = ctx.egress.clone();
3317 let mut streams = self
3318 .activity_streams
3319 .lock()
3320 .unwrap_or_else(|p| p.into_inner());
3321 streams.retain(|_, task| !task.is_finished());
3322 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3323 return Ok(vec![control_error_frame(
3324 &frame,
3325 "duplicate_subscription",
3326 "activity subscription correlation id already in use",
3327 )?]);
3328 }
3329 let task = tokio::spawn(async move {
3330 loop {
3331 let entry = if let Some(entry) = subscription.replay.pop_front() {
3332 Some(entry)
3333 } else {
3334 subscription.receiver.recv().await
3335 };
3336 let Some(entry) = entry else {
3337 break;
3338 };
3339 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3340 return;
3341 };
3342 let Ok(next) = Frame::build_with_version(
3343 response_version(&frame),
3344 FrameType::StreamData,
3345 control_flags(),
3346 0,
3347 0,
3348 frame.header.corr,
3349 body,
3350 ) else {
3351 return;
3352 };
3353 if sink.send(next).await.is_err() {
3354 return;
3355 }
3356 }
3357 if let Some(code) = subscription.terminal_code() {
3360 if let Ok(end) =
3361 control_error_frame(&frame, code, "activity subscriber fell behind")
3362 {
3363 let _ = sink.send(end).await;
3364 }
3365 }
3366 });
3367 streams.insert(key, task.abort_handle());
3368 Ok(Vec::new())
3369 }
3370
3371 async fn handle_activity_record(
3372 &self,
3373 ctx: &RouteCtx,
3374 frame: Frame,
3375 ) -> Result<Vec<Frame>, RouterError> {
3376 use subc_control::{
3377 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3378 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3379 };
3380 let result = async {
3381 let module = self
3382 .forwarding
3383 .module_id_for_connection(ctx.connection_id)
3384 .ok()
3385 .flatten()
3386 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3387 let request: ActivityRecordRequest =
3388 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3389 crate::activity::validate(&module, &request)?;
3390 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3391 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3392 let at_ms = std::time::SystemTime::now()
3393 .duration_since(std::time::UNIX_EPOCH)
3394 .unwrap_or_default()
3395 .as_millis() as u64;
3396 ledger.record(module, request, actor, at_ms).await
3397 }
3398 .await;
3399 match result {
3400 Ok(entry) => Ok(vec![control_response_body_frame(
3401 &frame,
3402 &ActivityRecordResponse { seq: entry.seq },
3403 "activity.record",
3404 )?]),
3405 Err(code) => Ok(vec![control_error_frame(
3406 &frame,
3407 code,
3408 "activity recording refused",
3409 )?]),
3410 }
3411 }
3412
3413 fn activity_actor(
3414 &self,
3415 connection: ConnectionId,
3416 module: &str,
3417 on: Option<&subc_control::ActivityOn>,
3418 ) -> Result<subc_control::ActivityActor, &'static str> {
3419 use crate::forwarding::{DataRoute, DataRouteState};
3420 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3421 let principal = |id: &str| Principal::Reserved {
3422 module_id: id.to_owned(),
3423 };
3424 let mut actor = ActivityActor {
3425 principal: principal(module),
3426 scope_owner: None,
3427 agent_id: None,
3428 run_id: None,
3429 flow_id: None,
3430 project_root: None,
3431 };
3432 let scope = match on {
3433 None => return Ok(actor),
3434 Some(ActivityOn::Scope(scope)) => {
3435 let selector = subc_protocol::scope::ScopeSelector {
3436 owner: principal(module),
3437 scope_ref: scope.scope_ref.clone(),
3438 scope_epoch: Some(scope.epoch),
3439 };
3440 self.scopes
3441 .read()
3442 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3443 .admit(&principal(module), module, &selector, true)
3444 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3445 }
3446 Some(ActivityOn::Route(route)) => {
3447 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3448 .forwarding
3449 .lookup_data_route(connection, route.channel, route.epoch)
3450 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3451 else {
3452 return Err(ACTIVITY_ACTOR_UNKNOWN);
3453 };
3454 if binding.module_id != module {
3455 return Err(ACTIVITY_ACTOR_UNKNOWN);
3456 }
3457 actor.principal = binding.principal.clone();
3458 actor.project_root = binding
3459 .project_root
3460 .as_ref()
3461 .map(|root| root.as_path().to_path_buf());
3462 let Some(bound) = &binding.scope else {
3463 return Ok(actor);
3464 };
3465 let owner = principal(&bound.owner);
3466 let selector = subc_protocol::scope::ScopeSelector {
3467 owner: owner.clone(),
3468 scope_ref: bound.scope_ref.clone(),
3469 scope_epoch: Some(bound.tag.scope_epoch),
3470 };
3471 let admitted = self
3472 .scopes
3473 .read()
3474 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3475 .admit(&owner, module, &selector, true)
3476 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3477 if admitted.tag != bound.tag {
3478 return Err(ACTIVITY_ACTOR_UNKNOWN);
3479 }
3480 admitted
3481 }
3482 };
3483 actor.scope_owner = Some(scope.stamp.owner);
3484 actor.agent_id = scope.stamp.attributes.agent_id;
3485 actor.run_id = scope.stamp.attributes.run_id;
3486 actor.flow_id = scope.stamp.attributes.flow_id;
3487 Ok(actor)
3488 }
3489
3490 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3491 self.refresh_capability_requirements();
3492 let mut counters = self.counters.snapshot();
3498 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3499 obj.insert(
3500 "activity_unparseable_lines".into(),
3501 ledger.read_errors().into(),
3502 );
3503 obj.insert(
3504 "activity_lifecycle_append_failures".into(),
3505 ledger.lifecycle_failures().into(),
3506 );
3507 obj.insert(
3508 "activity_started_digest_skips".into(),
3509 ledger.lifecycle_digest_skips().into(),
3510 );
3511 }
3512 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3513 self.forwarding.client_route_concentration(),
3514 counters.as_object_mut(),
3515 ) {
3516 obj.insert(
3517 "client_connections_with_routes".into(),
3518 connections_with_routes.into(),
3519 );
3520 obj.insert("max_routes_on_one_connection".into(), max.into());
3521 }
3522 if let (Some(open_breakers), Some(obj)) = (
3528 self.route_bind_breakers.open_snapshot(),
3529 counters.as_object_mut(),
3530 ) {
3531 obj.insert("route_bind_breakers_open".into(), open_breakers);
3532 }
3533 let response = ClientControlResponse::ServerDescribe {
3534 protocol_ver: PROTOCOL_VERSION,
3535 subc_ops: subc_ops(),
3536 capabilities: self.subc_capabilities.as_ref().to_vec(),
3537 connected_clients: self.connected_clients.count(),
3538 counters: Some(counters),
3539 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3540 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3541 capability_requirements: self.capability_requirement_statuses(),
3542 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3543 };
3544 Ok(vec![control_response_body_frame(
3545 &frame,
3546 &response,
3547 "ClientControlResponse::ServerDescribe",
3548 )?])
3549 }
3550
3551 fn handle_catalog_list(
3552 &self,
3553 frame: Frame,
3554 module_id: Option<String>,
3555 ) -> Result<Vec<Frame>, RouterError> {
3556 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3557 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3558 })?;
3559 let entries = modules
3560 .into_iter()
3561 .filter(|registration| {
3562 module_id
3563 .as_deref()
3564 .map(|wanted| registration.manifest.module_id == wanted)
3565 .unwrap_or(true)
3566 })
3567 .map(|registration| {
3568 let not_ready = self.not_ready_reason(®istration);
3569 let roles = registration.manifest.provides;
3570 CatalogEntry::new(
3571 registration.manifest.module_id,
3572 roles,
3573 registration.control_ops,
3574 )
3575 .with_ready(not_ready.is_none())
3576 .with_not_ready(not_ready)
3577 .with_module_version(Some(registration.manifest.module_version))
3578 .with_capabilities(registration.manifest.capabilities)
3579 .with_self_signals(registration.manifest.self_signals)
3580 .with_events(registration.manifest.events)
3581 })
3582 .collect();
3583 let response = ClientControlResponse::CatalogList {
3584 generation,
3585 modules: entries,
3586 subc_ops: subc_ops(),
3587 };
3588 Ok(vec![control_response_body_frame(
3589 &frame,
3590 &response,
3591 "ClientControlResponse::CatalogList",
3592 )?])
3593 }
3594
3595 fn route_open_principal(
3596 &self,
3597 frame: &Frame,
3598 consumer_identity: Option<ConsumerIdentity>,
3599 ) -> Result<Result<Principal, Frame>, RouterError> {
3600 let Some(consumer_identity) = consumer_identity else {
3601 return Ok(Ok(Principal::Direct));
3602 };
3603
3604 if self.supervisor.spawned_consumer_authorized(
3605 &consumer_identity.module_id,
3606 &consumer_identity.launch_nonce,
3607 ) {
3608 return Ok(Ok(Principal::Reserved {
3609 module_id: consumer_identity.module_id,
3610 }));
3611 }
3612
3613 Ok(Err(control_error_frame(
3614 frame,
3615 "bad_consumer_identity",
3616 format!(
3617 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3618 consumer_identity.module_id
3619 ),
3620 )?))
3621 }
3622
3623 fn route_open_refusal_frame(
3630 &self,
3631 ctx: &RouteCtx,
3632 frame: &Frame,
3633 module_id: &str,
3634 reason: &'static str,
3635 code: &'static str,
3636 message: impl Into<String>,
3637 ) -> Result<Frame, RouterError> {
3638 self.observe_route_open_refusal(ctx, module_id, reason, code);
3639 control_error_frame(frame, code, message.into())
3640 }
3641
3642 fn route_open_breaker_refusal_frame(
3660 &self,
3661 ctx: &RouteCtx,
3662 frame: &Frame,
3663 module_id: &str,
3664 consecutive_timeouts: u32,
3665 retry_in: Duration,
3666 probe_in_flight: bool,
3667 ) -> Result<Frame, RouterError> {
3668 self.counters
3669 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3670 debug!(
3671 target: "control",
3672 code = "module_timeout",
3673 module_id = ?module_id,
3674 connection_id = ctx.connection_id.get(),
3675 consecutive_timeouts,
3676 retry_in_ms = retry_in.as_millis() as u64,
3677 probe_in_flight,
3678 "route.open refused by open bind-relay breaker"
3679 );
3680 let detail = if probe_in_flight {
3687 "one test route is already being tried; retry once it settles".to_string()
3688 } else {
3689 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3690 };
3691 control_error_frame(
3692 frame,
3693 "module_timeout",
3694 format!(
3695 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3696 timed out in a row); {detail}; its established routes are unaffected"
3697 ),
3698 )
3699 }
3700
3701 fn observe_route_open_refusal(
3714 &self,
3715 ctx: &RouteCtx,
3716 module_id: &str,
3717 reason: &'static str,
3718 code: &'static str,
3719 ) {
3720 self.counters.increment_route_open_refused(code);
3721 info!(
3722 target: "control",
3723 code,
3724 reason,
3725 module_id = ?module_id,
3726 connection_id = ctx.connection_id.get(),
3727 "route.open refused"
3728 );
3729 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3730 self.route_outages.record_not_serving(module_id, reason);
3731 }
3732 }
3733
3734 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3779 self.route_outages.record_accepted(module_id);
3780 self.counters.increment_route_open_accepted(principal);
3781 info!(
3782 target: "control",
3783 principal,
3784 module_id,
3785 connection_id = ctx.connection_id.get(),
3786 "route.open accepted"
3787 );
3788 }
3789
3790 fn supervised_absent_route_open_refusal_frame(
3791 &self,
3792 ctx: &RouteCtx,
3793 frame: &Frame,
3794 module_id: &str,
3795 code: &'static str,
3796 status: &crate::supervise::ModuleStatus,
3797 ) -> Result<Frame, RouterError> {
3798 self.counters.increment_route_open_refused(code);
3799 info!(
3800 target: "control",
3801 code,
3802 reason = "supervised_not_registered",
3803 module_id = ?module_id,
3804 connection_id = ctx.connection_id.get(),
3805 state = %status.state,
3806 enabled = status.enabled,
3807 live = status.live,
3808 "route.open refused"
3809 );
3810 self.route_outages
3814 .record_not_serving(module_id, "supervised_not_registered");
3815 let message = if let Some(pid) = status.waiting_for_previous_pid {
3816 format!("waiting: previous instance (pid {pid}) is still running but hidden; it cannot be identified")
3817 } else {
3818 format!(
3819 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
3820 status.state, status.enabled, status.live
3821 )
3822 };
3823 control_error_frame(frame, code, message)
3824 }
3825
3826 async fn handle_route_open(
3827 &self,
3828 ctx: &RouteCtx,
3829 frame: Frame,
3830 request: RouteOpenRequest,
3831 ) -> Result<Vec<Frame>, RouterError> {
3832 let RouteOpenRequest {
3833 target,
3834 mut identity,
3835 consumer_identity,
3836 consumer_capabilities,
3837 role_versions,
3838 admission_facts,
3839 scope,
3840 } = request;
3841 let target_module_id = target_module_id(&target).to_string();
3842 if self
3843 .registry
3844 .get_module_by_connection(ctx.connection_id)
3845 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3846 .is_some()
3847 {
3848 return Ok(vec![control_error_frame(
3849 &frame,
3850 "invalid_request",
3851 "module connections cannot open client routes",
3852 )?]);
3853 }
3854 debug!(
3855 connection_id = ctx.connection_id.get(),
3856 corr = frame.header.corr,
3857 module_id = %target_module_id,
3858 "handling route.open"
3859 );
3860
3861 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
3867 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
3868 self.observe_route_open_refusal(
3869 ctx,
3870 &target_module_id,
3871 "invalid_role_versions",
3872 error_codes::INVALID_REQUEST,
3873 );
3874 return Ok(vec![control_error_body_frame(
3875 &frame,
3876 ErrorBody {
3877 code: error_codes::INVALID_REQUEST.to_string(),
3878 message: error.to_string(),
3879 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
3880 },
3881 )?]);
3882 }
3883
3884 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
3885 if status.waiting_for_previous_pid.is_some() {
3886 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3887 ctx,
3888 &frame,
3889 &target_module_id,
3890 "module_warming",
3891 &status,
3892 )?]);
3893 }
3894 }
3895
3896 let Some(registration) = self
3915 .registry
3916 .get_module(&target_module_id)
3917 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3918 else {
3919 if let Some((status, warming)) =
3920 self.supervisor_status(&target_module_id, frame.header.corr)?
3921 {
3922 if status.protocol == ModuleProtocol::None {
3929 return Ok(vec![self.route_open_refusal_frame(
3930 ctx,
3931 &frame,
3932 &target_module_id,
3933 "protocol_none",
3934 error_codes::MODULE_NO_PROTOCOL,
3935 format!(
3936 "module_id '{target_module_id}' is declared protocol: none; \
3937 it speaks no subc wire and serves no routes"
3938 ),
3939 )?]);
3940 }
3941 let code = if warming {
3942 "module_warming"
3943 } else {
3944 "target_unavailable"
3945 };
3946 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
3947 ctx,
3948 &frame,
3949 &target_module_id,
3950 code,
3951 &status,
3952 )?]);
3953 }
3954 if let Some(removed_ago_ms) =
3955 self.supervisor.removal_tombstone_age_ms(&target_module_id)
3956 {
3957 return Ok(vec![self.route_open_refusal_frame(
3958 ctx,
3959 &frame,
3960 &target_module_id,
3961 "removed",
3962 error_codes::MODULE_REMOVED,
3963 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
3964 )?]);
3965 }
3966 return Ok(vec![self.route_open_refusal_frame(
3967 ctx,
3968 &frame,
3969 &target_module_id,
3970 "not_registered",
3971 error_codes::UNKNOWN_MODULE,
3972 format!("module_id '{target_module_id}' is not registered"),
3973 )?]);
3974 };
3975
3976 if !registration.ready {
3980 self.counters
3981 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
3982 info!(
3983 target: "control",
3984 code = error_codes::MODULE_WARMING,
3985 module_id = ?target_module_id,
3986 connection_id = ctx.connection_id.get(),
3987 reason = "declared_not_ready",
3988 "route.open refused"
3989 );
3990 self.route_outages
3993 .record_not_serving(&target_module_id, "declared_not_ready");
3994 return Ok(vec![control_error_body_frame(
3995 &frame,
3996 ErrorBody {
3997 code: error_codes::MODULE_WARMING.to_string(),
3998 message: format!(
3999 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4000 ),
4001 detail: Some(serde_json::json!({
4002 "reason": "declared_not_ready"
4003 })),
4004 },
4005 )?]);
4006 }
4007
4008 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4027 self.counters
4028 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4029 info!(
4030 target: "control",
4031 code = error_codes::MODULE_WARMING,
4032 module_id = ?target_module_id,
4033 connection_id = ctx.connection_id.get(),
4034 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4035 capability = %capability,
4036 "route.open refused"
4037 );
4038 return Ok(vec![control_error_body_frame(
4039 &frame,
4040 ErrorBody {
4041 code: error_codes::MODULE_WARMING.to_string(),
4042 message: format!(
4043 "module_id '{target_module_id}' requires capability '{capability}', \
4044 which no registered module provides; retry"
4045 ),
4046 detail: Some(serde_json::json!({
4047 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4048 "capability": capability,
4049 })),
4050 },
4051 )?]);
4052 }
4053
4054 if !target_has_required_role(&target, ®istration.manifest.provides) {
4055 return Ok(vec![self.route_open_refusal_frame(
4056 ctx,
4057 &frame,
4058 &target_module_id,
4059 "role_not_provided",
4060 "target_unavailable",
4061 format!("module_id '{target_module_id}' does not provide the requested target"),
4062 )?]);
4063 }
4064
4065 if registration.state != ChannelState::Active {
4066 return Ok(vec![self.route_open_refusal_frame(
4067 ctx,
4068 &frame,
4069 &target_module_id,
4070 "registration_not_active",
4071 "target_unavailable",
4072 format!("module_id '{target_module_id}' is not active"),
4073 )?]);
4074 }
4075
4076 if self
4077 .forwarding
4078 .module_is_draining(&target_module_id)
4079 .map_err(RouterError::Forwarding)?
4080 {
4081 return Ok(vec![self.route_open_refusal_frame(
4082 ctx,
4083 &frame,
4084 &target_module_id,
4085 "reloading",
4086 "module_reloading",
4087 format!("module_id '{target_module_id}' is reloading"),
4088 )?]);
4089 }
4090
4091 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4092 process_liveness.process_live(&target_module_id) == Some(false)
4093 }) {
4094 if process_liveness.process_replacing(&target_module_id) {
4101 return Ok(vec![self.route_open_refusal_frame(
4102 ctx,
4103 &frame,
4104 &target_module_id,
4105 "reloading",
4106 "module_reloading",
4107 format!("module_id '{target_module_id}' is reloading"),
4108 )?]);
4109 }
4110 return Ok(vec![self.route_open_refusal_frame(
4111 ctx,
4112 &frame,
4113 &target_module_id,
4114 "supervisor_not_live",
4115 "target_unavailable",
4116 format!("module_id '{target_module_id}' is not live"),
4117 )?]);
4118 }
4119
4120 if !self
4121 .forwarding
4122 .has_live_module_connection(&target_module_id)
4123 .map_err(RouterError::Forwarding)?
4124 {
4125 return Ok(vec![self.route_open_refusal_frame(
4126 ctx,
4127 &frame,
4128 &target_module_id,
4129 "no_forwarding_connection",
4130 "target_unavailable",
4131 format!("module_id '{target_module_id}' has no live forwarding connection"),
4132 )?]);
4133 }
4134
4135 if let Some(error) =
4136 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4137 {
4138 self.observe_route_open_refusal(
4139 ctx,
4140 &target_module_id,
4141 "op_not_allowed",
4142 "op_not_allowed",
4143 );
4144 return Ok(vec![error]);
4145 }
4146
4147 let principal = match self.route_open_principal(&frame, consumer_identity)? {
4148 Ok(principal) => principal,
4149 Err(error) => {
4150 self.observe_route_open_refusal(
4151 ctx,
4152 &target_module_id,
4153 "bad_consumer_identity",
4154 "bad_consumer_identity",
4155 );
4156 return Ok(vec![error]);
4157 }
4158 };
4159
4160 if let Principal::Reserved {
4164 module_id: opening_module_id,
4165 } = &principal
4166 {
4167 if let Some(opening_registration) = self
4168 .registry
4169 .get_module(opening_module_id)
4170 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4171 {
4172 if let Some(capability) =
4173 denied_capability(&opening_registration.manifest, ®istration.manifest)
4174 {
4175 warn!(
4176 opening_module_id,
4177 target_module_id,
4178 capability,
4179 "refusing route.open because an attested capability deny edge matches"
4180 );
4181 return Ok(vec![self.route_open_refusal_frame(
4182 ctx,
4183 &frame,
4184 &target_module_id,
4185 "capability_deny_edge",
4186 "capability_forbidden",
4187 format!(
4188 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4189 ),
4190 )?]);
4191 }
4192 }
4193 }
4194
4195 if admission_facts.is_some() {
4196 let carrier_matches = matches!(
4197 &principal,
4198 Principal::Reserved { module_id }
4199 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4200 );
4201 if !carrier_matches {
4202 return Ok(vec![self.route_open_refusal_frame(
4203 ctx,
4204 &frame,
4205 &target_module_id,
4206 "admission_facts_carrier_not_permitted",
4207 "admission_facts_not_permitted",
4208 "admission facts may only be carried by the configured reserved module",
4209 )?]);
4210 }
4211
4212 let target_allowed = self
4213 .admission_facts_targets
4214 .as_ref()
4215 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4216 if !target_allowed {
4217 return Ok(vec![self.route_open_refusal_frame(
4218 ctx,
4219 &frame,
4220 &target_module_id,
4221 "admission_facts_target_not_listed",
4222 "admission_facts_target_not_allowed",
4223 format!(
4224 "admission facts are not permitted for target module_id '{target_module_id}'"
4225 ),
4226 )?]);
4227 }
4228
4229 }
4233
4234 let (bound_scope, mut scope_stamp) = match scope {
4240 None => (None, None),
4241 Some(selector) => {
4242 let owner_configured = match &selector.owner {
4243 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4249 _ => false,
4250 };
4251 let admitted = self
4252 .scopes
4253 .read()
4254 .unwrap_or_else(|poisoned| poisoned.into_inner())
4255 .admit(&principal, &target_module_id, &selector, owner_configured)
4256 .and_then(|admission| {
4257 crate::scopes::check_target_flow_support(
4258 &admission.stamp,
4259 &target_module_id,
4260 registration.manifest.capabilities.as_ref(),
4261 )?;
4262 crate::scopes::check_target_agent_run_support(
4263 &admission.stamp,
4264 &target_module_id,
4265 registration.manifest.capabilities.as_ref(),
4266 )?;
4267 Ok(admission)
4268 });
4269 match admitted {
4270 Ok(admission) => (
4271 Some(BoundScope {
4272 owner: admission.owner,
4273 scope_ref: admission.stamp.scope_ref.clone(),
4274 tag: admission.tag,
4275 }),
4276 Some(admission.stamp),
4277 ),
4278 Err(refusal) => {
4279 return Ok(vec![self.route_open_refusal_frame(
4280 ctx,
4281 &frame,
4282 &target_module_id,
4283 refusal.code,
4284 refusal.code,
4285 refusal.message,
4286 )?]);
4287 }
4288 }
4289 }
4290 };
4291
4292 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4314 Ok(project_root) => project_root,
4315 Err(err) => {
4316 return Ok(vec![control_error_frame(
4317 &frame,
4318 "invalid_project_root",
4319 err.to_string(),
4320 )?])
4321 }
4322 };
4323 identity.project_root = project_root.as_path().to_path_buf();
4324
4325 let _concurrency_guard = match self
4336 .route_bind_concurrency
4337 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4338 {
4339 Ok(guard) => guard,
4340 Err(in_flight) => {
4341 return Ok(vec![self.route_open_target_capacity_refusal(
4342 ctx,
4343 &frame,
4344 &target_module_id,
4345 in_flight,
4346 )?]);
4347 }
4348 };
4349
4350 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4353 RouteBindAdmission::Admitted { guard, probe } => {
4354 if probe {
4355 info!(
4356 module_id = %target_module_id,
4357 connection_id = ctx.connection_id.get(),
4358 "route.bind breaker half-open: admitting one probe"
4359 );
4360 }
4361 guard
4362 }
4363 RouteBindAdmission::Refused {
4364 consecutive_timeouts,
4365 retry_in,
4366 probe_in_flight,
4367 } => {
4368 return Ok(vec![self.route_open_breaker_refusal_frame(
4369 ctx,
4370 &frame,
4371 &target_module_id,
4372 consecutive_timeouts,
4373 retry_in,
4374 probe_in_flight,
4375 )?]);
4376 }
4377 };
4378
4379 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4384 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4385 let pending = match self
4386 .forwarding
4387 .begin_route_bind_relay_for(
4388 ctx.connection_id,
4389 ctx.egress.clone(),
4390 response_version(&frame),
4391 frame.header.corr,
4392 &target_module_id,
4393 principal.clone(),
4394 bound_scope,
4395 Some(project_root),
4396 relay_deadline,
4397 )
4398 .await
4399 {
4400 Ok(pending) => pending,
4401 Err(err) => {
4402 return Ok(vec![self.route_open_refusal_frame(
4403 ctx,
4404 &frame,
4405 &target_module_id,
4406 "relay_reservation_failed",
4407 forwarding_error_code(&err),
4408 err.to_string(),
4409 )?])
4410 }
4411 };
4412 let crate::forwarding::PendingRouteBindRelay {
4413 endpoint,
4414 module_sink,
4415 negotiated_ver,
4416 client_channel,
4417 client_epoch,
4418 module_channel,
4419 module_epoch,
4420 corr: relay_corr,
4421 receiver,
4422 } = pending;
4423 let mut reservation =
4424 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4425
4426 let relay_registration = self
4431 .registry
4432 .get_module_by_connection(endpoint.connection_id)
4433 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4434 let relay_capabilities = relay_registration
4435 .as_ref()
4436 .and_then(|registration| registration.manifest.capabilities.as_ref());
4437 if let Some(stamp) = scope_stamp
4438 .as_ref()
4439 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4440 {
4441 if let Err(refusal) = crate::scopes::check_target_flow_support(
4442 stamp,
4443 &target_module_id,
4444 relay_capabilities,
4445 )
4446 .and_then(|()| {
4447 crate::scopes::check_target_agent_run_support(
4448 stamp,
4449 &target_module_id,
4450 relay_capabilities,
4451 )
4452 }) {
4453 reservation.release_and_disarm();
4454 return Ok(vec![self.route_open_refusal_frame(
4455 ctx,
4456 &frame,
4457 &target_module_id,
4458 refusal.code,
4459 refusal.code,
4460 refusal.message,
4461 )?]);
4462 }
4463 }
4464
4465 let call_mode = if relay_capabilities
4468 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4469 {
4470 Some(derive_call_mode(scope_stamp.as_ref()))
4471 } else {
4472 if let Some(stamp) = scope_stamp.as_mut() {
4473 stamp.expires_at_ms = None;
4474 }
4475 None
4476 };
4477
4478 debug!(
4479 connection_id = ctx.connection_id.get(),
4480 client_channel,
4481 client_epoch,
4482 module_channel,
4483 module_epoch,
4484 "reserved route handle pair"
4485 );
4486 let principal_label = principal_label(&principal);
4489 let relay = ModuleControlRequest::RouteBind {
4490 route_channel: module_channel,
4491 epoch: module_epoch,
4492 target,
4493 identity,
4494 principal: Some(principal),
4495 consumer_capabilities,
4496 role_versions,
4497 admission_facts,
4498 scope: scope_stamp,
4499 call_mode,
4500 };
4501 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4502 RouterError::backend(
4503 0,
4504 frame.header.corr,
4505 format!("failed to encode route.bind request: {err}"),
4506 )
4507 })?;
4508 let relay_frame = Frame::build_with_version(
4509 negotiated_ver,
4510 FrameType::Request,
4511 control_flags(),
4512 0,
4513 0,
4514 relay_corr,
4515 relay_body,
4516 )
4517 .map_err(RouterError::FrameBuild)?;
4518
4519 if let Err(err) = module_sink.send(relay_frame).await {
4520 reservation.release_and_disarm();
4521 return Ok(vec![self.route_open_refusal_frame(
4522 ctx,
4523 &frame,
4524 &target_module_id,
4525 "relay_send_failed",
4526 "target_unavailable",
4527 err.to_string(),
4528 )?]);
4529 }
4530
4531 #[cfg(test)]
4532 if let Some(hold) = &self.route_bind_relay_sent_hold {
4533 hold.wait().await;
4537 hold.wait().await;
4538 }
4539
4540 if self
4545 .forwarding
4546 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4547 .map_err(RouterError::Forwarding)?
4548 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4549 {
4550 self.send_abandoned_route_bind_goodbye(
4551 &module_sink,
4552 negotiated_ver,
4553 module_channel,
4554 module_epoch,
4555 );
4556 }
4557
4558 match timeout_at(relay_deadline, receiver).await {
4559 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4560 reservation.disarm();
4561 if breaker.record_accepted() {
4562 info!(
4563 module_id = %target_module_id,
4564 "route.bind breaker closed: the probe was accepted"
4565 );
4566 }
4567 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4568 Ok(Vec::new())
4569 }
4570 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4571 reservation.release_and_disarm();
4572 breaker.record_inconclusive();
4576 let scope_code = match body.code.as_str() {
4580 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4581 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4582 _ => None,
4583 };
4584 if let Some(code) = scope_code {
4585 self.observe_route_open_refusal(
4586 ctx,
4587 &target_module_id,
4588 "scope_changed_before_commit",
4589 code,
4590 );
4591 return Ok(vec![control_error_body_frame(&frame, body)?]);
4592 }
4593 self.counters
4594 .increment_route_open_refused("module_rejected");
4595 info!(
4596 target: "control",
4597 code = "module_rejected",
4598 module_code = ?body.code,
4599 module_id = ?target_module_id,
4600 connection_id = ctx.connection_id.get(),
4601 "route.open refused"
4602 );
4603 Ok(vec![control_error_body_frame(&frame, body)?])
4604 }
4605 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4606 reservation.release_and_disarm();
4607 breaker.record_inconclusive();
4608 tracing::warn!(
4615 module_id = %target_module_id,
4616 "route.bind relay abandoned: {message}"
4617 );
4618 Ok(vec![self.route_open_refusal_frame(
4619 ctx,
4620 &frame,
4621 &target_module_id,
4622 "relay_abandoned",
4623 "target_unavailable",
4624 message,
4625 )?])
4626 }
4627 Ok(Err(_)) => {
4628 reservation.release_and_disarm();
4629 breaker.record_inconclusive();
4630 Ok(vec![self.route_open_refusal_frame(
4631 ctx,
4632 &frame,
4633 &target_module_id,
4634 "relay_waiter_canceled",
4635 "target_unavailable",
4636 "route.bind relay waiter was canceled before the module responded",
4637 )?])
4638 }
4639 Err(_) => {
4640 reservation.release_and_disarm();
4641 if let Some(opened) = breaker.record_timeout(
4645 self.route_bind_breaker_threshold,
4646 self.route_bind_breaker_cooldown,
4647 ) {
4648 warn!(
4649 module_id = %target_module_id,
4650 consecutive_timeouts = opened.consecutive_timeouts,
4651 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4652 reopened_after_probe = opened.reopened_after_probe,
4653 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4654 );
4655 }
4656 tracing::warn!(
4664 module_id = %target_module_id,
4665 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4666 "route.bind relay timed out: module did not ack within budget"
4667 );
4668 Ok(vec![self.route_open_refusal_frame(
4669 ctx,
4670 &frame,
4671 &target_module_id,
4672 "relay_timed_out",
4673 "module_timeout",
4674 format!(
4675 "module_id '{target_module_id}' did not answer route.bind within {:?}",
4676 route_bind_relay_timeout
4677 ),
4678 )?])
4679 }
4680 }
4681 }
4682
4683 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4684 let response = ClientControlResponse::SupervisorSpawnSnapshot {
4685 snapshot: self.supervisor.spawn_snapshot(),
4686 };
4687 Ok(vec![control_response_body_frame(
4688 &frame,
4689 &response,
4690 "ClientControlResponse::SupervisorSpawnSnapshot",
4691 )?])
4692 }
4693
4694 fn handle_supervisor_spawn_subscribe(
4695 &self,
4696 ctx: &RouteCtx,
4697 frame: Frame,
4698 since: Option<SpawnCursor>,
4699 ) -> Result<Vec<Frame>, RouterError> {
4700 match self.supervisor.subscribe_spawns(
4701 ctx.connection_id,
4702 frame.header.corr,
4703 response_version(&frame),
4704 since,
4705 ctx.egress.clone(),
4706 ) {
4707 Ok(()) => Ok(Vec::new()),
4708 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4709 Ok(vec![control_error_body_frame(
4710 &frame,
4711 ErrorBody {
4712 code: "spawn_cursor_incarnation_mismatch".to_string(),
4713 message: "spawn cursor belongs to a different daemon incarnation"
4714 .to_string(),
4715 detail: Some(serde_json::json!({
4716 "current_daemon_incarnation": current
4717 })),
4718 },
4719 )?])
4720 }
4721 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4722 &frame,
4723 ErrorBody {
4724 code: "spawn_cursor_too_old".to_string(),
4725 message: "spawn cursor predates the retained event ring".to_string(),
4726 detail: Some(serde_json::json!({
4727 "oldest_retained_cursor": oldest
4728 })),
4729 },
4730 )?]),
4731 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4732 0,
4733 frame.header.corr,
4734 format!("failed to open supervisor spawn subscription: {error}"),
4735 )),
4736 }
4737 }
4738
4739 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4740 let generation = self
4741 .registry
4742 .generation()
4743 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4744 let mut modules = Vec::new();
4745 for module in self.supervisor.list() {
4746 let status = module.status_for_control("list").map_err(|err| {
4747 RouterError::backend(
4748 0,
4749 frame.header.corr,
4750 format!("failed to read supervisor status: {err}"),
4751 )
4752 })?;
4753 let (configured, _) = module.configuration().map_err(|err| {
4754 RouterError::backend(
4755 0,
4756 frame.header.corr,
4757 format!("failed to read module configuration: {err}"),
4758 )
4759 })?;
4760 let image = module.running_image_agreement().await;
4762 let resources = Some(module.child_resource_usage());
4765 let pending_reload = Some(reload_verdict(
4766 &configured.program,
4767 status.spawned_from.as_deref(),
4768 image,
4769 ));
4770 modules.push(
4773 SupervisorEntry::new(
4774 status.module_id,
4775 status.state.to_string(),
4776 status.enabled,
4777 status.live,
4778 status.health.status,
4779 )
4780 .with_launch_nonce_env(Some(!cfg!(unix)))
4781 .with_launch_nonce_source(status.launch_nonce_source)
4782 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
4783 .with_protocol(status.protocol)
4784 .with_pending_reload(pending_reload)
4785 .with_last_probe_ms(status.health.last_probe_ms)
4786 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
4787 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
4788 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
4789 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
4790 .with_restart_count(Some(status.restart_count))
4791 .with_max_restarts(Some(status.max_restarts))
4792 .with_lifetime_restarts(Some(status.lifetime_restarts))
4793 .with_spawn_generation(Some(status.spawn_generation))
4794 .with_restart_window_secs(Some(status.restart_window.as_secs()))
4795 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
4796 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
4797 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
4798 .with_resources(resources),
4799 );
4800 }
4801 let response = ClientControlResponse::SupervisorList {
4802 generation,
4803 modules,
4804 };
4805 Ok(vec![control_response_body_frame(
4806 &frame,
4807 &response,
4808 "ClientControlResponse::SupervisorList",
4809 )?])
4810 }
4811
4812 fn handle_supervisor_stderr_tail(
4813 &self,
4814 frame: Frame,
4815 module_id: String,
4816 max_lines: Option<u32>,
4817 max_bytes: Option<u32>,
4818 ) -> Result<Vec<Frame>, RouterError> {
4819 let Some(module) = self.supervisor.get(&module_id) else {
4820 return Ok(vec![control_error_frame(
4821 &frame,
4822 "unknown_module",
4823 format!("module_id '{module_id}' is not supervised"),
4824 )?]);
4825 };
4826
4827 let snapshot = module.stderr_tail(
4828 max_lines.map(|value| value as usize),
4829 max_bytes.map(|value| value as usize),
4830 );
4831
4832 let response = ClientControlResponse::SupervisorStderrTail {
4833 module_id,
4834 tail: StderrTail {
4835 capture: match snapshot.capture {
4836 CaptureState::Captured => StderrCaptureState::Captured,
4837 CaptureState::Incomplete { reason } => {
4838 StderrCaptureState::Incomplete { reason }
4839 }
4840 CaptureState::NotCaptured { reason } => {
4841 StderrCaptureState::NotCaptured { reason }
4842 }
4843 },
4844 entries: snapshot
4845 .entries
4846 .into_iter()
4847 .map(|entry| match entry {
4848 TailEntry::Line {
4849 text,
4850 truncated,
4851 at_ms,
4852 } => StderrTailEntry::Line {
4853 text,
4854 truncated,
4855 at_ms,
4856 },
4857 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
4858 })
4859 .collect(),
4860 dropped_lines: snapshot.dropped_lines,
4861 },
4862 };
4863 Ok(vec![control_response_body_frame(
4864 &frame,
4865 &response,
4866 "ClientControlResponse::SupervisorStderrTail",
4867 )?])
4868 }
4869
4870 async fn handle_supervisor_terminals(
4871 &self,
4872 frame: Frame,
4873 module_id: String,
4874 ) -> Result<Vec<Frame>, RouterError> {
4875 let Some(module) = self.supervisor.get(&module_id) else {
4876 return Ok(vec![control_error_frame(
4877 &frame,
4878 "unknown_module",
4879 format!("module_id '{module_id}' is not supervised"),
4880 )?]);
4881 };
4882
4883 let terminals = module
4886 .read_durable_terminal_history()
4887 .await
4888 .map_err(|error| {
4889 RouterError::backend(
4890 0,
4891 frame.header.corr,
4892 format!("failed to read terminal history: {error}"),
4893 )
4894 })?;
4895 let response = ClientControlResponse::SupervisorTerminals {
4896 module_id,
4897 terminals,
4898 };
4899 Ok(vec![control_response_body_frame(
4900 &frame,
4901 &response,
4902 "ClientControlResponse::SupervisorTerminals",
4903 )?])
4904 }
4905
4906 fn handle_supervisor_routes(
4907 &self,
4908 frame: Frame,
4909 module_id: Option<String>,
4910 ) -> Result<Vec<Frame>, RouterError> {
4911 let modules = self
4912 .forwarding
4913 .route_census(module_id.as_deref())
4914 .map_err(RouterError::Forwarding)?
4915 .into_iter()
4916 .map(|(module_id, routes)| SupervisorRouteModule {
4917 module_id,
4918 routes: routes
4919 .into_iter()
4920 .map(|route| SupervisorRoute {
4921 consumer: match route.principal {
4922 Principal::Reserved { module_id } => {
4923 SupervisorRouteConsumer::Reserved { module_id }
4924 }
4925 Principal::Direct | Principal::Unverified => {
4926 SupervisorRouteConsumer::Direct {
4927 connection_id: route.goodbye_target.connection_id.get(),
4928 }
4929 }
4930 },
4931 age_ms: Instant::now()
4932 .saturating_duration_since(route.bound_at)
4933 .as_millis()
4934 .try_into()
4935 .unwrap_or(u64::MAX),
4936 draining: route.draining,
4937 drain_reason: route.drain_reason,
4938 })
4939 .collect(),
4940 })
4941 .collect();
4942 let response = ClientControlResponse::SupervisorRoutes { modules };
4943 Ok(vec![control_response_body_frame(
4944 &frame,
4945 &response,
4946 "ClientControlResponse::SupervisorRoutes",
4947 )?])
4948 }
4949
4950 async fn handle_supervisor_provenance(
4951 &self,
4952 frame: Frame,
4953 module_id: Option<String>,
4954 ) -> Result<Vec<Frame>, RouterError> {
4955 let mut selected = if let Some(module_id) = module_id {
4956 let Some(module) = self.supervisor.get(&module_id) else {
4957 return Ok(vec![control_error_frame(
4958 &frame,
4959 "unknown_module",
4960 format!("module_id '{module_id}' is not supervised"),
4961 )?]);
4962 };
4963 vec![module]
4964 } else {
4965 self.supervisor.list()
4966 };
4967
4968 let mut modules = Vec::with_capacity(selected.len());
4969 for module in selected.drain(..) {
4970 let (status, observed_image) = module
4971 .status_and_running_image_agreement()
4972 .await
4973 .map_err(|err| {
4974 RouterError::backend(
4975 0,
4976 frame.header.corr,
4977 format!("failed to read supervisor status: {err}"),
4978 )
4979 })?;
4980 let module_declared = self
4981 .registry
4982 .get_module(&status.module_id)
4983 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4984 .and_then(|registration| registration.manifest.provenance)
4985 .map(|build| ModuleDeclaredProvenance::Reported { build })
4986 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
4987 #[cfg(test)]
4988 let running_image = match &self.provenance_probe_override {
4989 Some(result) => result.clone(),
4990 None => observed_image,
4991 };
4992 #[cfg(not(test))]
4993 let running_image = observed_image;
4994 modules.push(SupervisorModuleProvenance {
4995 module_id: status.module_id,
4996 module_declared,
4997 daemon_observed: SupervisorObservedProcess {
4998 launch_nonce_source: status.launch_nonce_source,
4999 pid: status.pid,
5000 spawned_at_ms: status.spawned_at_ms,
5001 spawned_from: status.spawned_from,
5002 running_image,
5003 },
5004 });
5005 }
5006 let daemon = SupervisorDaemonProvenance {
5007 daemon_build: self.daemon_provenance.build.clone(),
5008 daemon_observed: DaemonObservedProcess {
5009 pid: self.daemon_provenance.pid,
5010 started_at_ms: self
5011 .daemon_provenance
5012 .start_clock
5013 .map(|clock| clock.started_at_ms())
5014 .or(self.daemon_provenance.started_at_ms),
5015 running_image: self
5016 .daemon_provenance
5017 .probe
5018 .observe(
5019 self.daemon_provenance.pid,
5020 self.daemon_provenance.executable_path.as_deref(),
5021 self.daemon_provenance.executable_identity.clone(),
5022 self.daemon_provenance.process_start_time,
5023 )
5024 .await,
5025 },
5026 };
5027 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5028 Ok(vec![control_response_body_frame(
5029 &frame,
5030 &response,
5031 "ClientControlResponse::SupervisorProvenance",
5032 )?])
5033 }
5034
5035 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5036 self.refresh_capability_requirements();
5037 let generation = self
5038 .registry
5039 .generation()
5040 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5041 let modules = self
5042 .supervisor
5043 .list()
5044 .into_iter()
5045 .map(|module| {
5046 let status = module.status_for_control("health").map_err(|err| {
5047 RouterError::backend(
5048 0,
5049 frame.header.corr,
5050 format!("failed to read supervisor health: {err}"),
5051 )
5052 })?;
5053 let module_id = status.module_id;
5054 let capability_detail = self
5055 .capability_evaluator
5056 .required_problem_detail(&module_id);
5057 Ok(SupervisorHealthEntry {
5058 module_id,
5059 status: status.health.status,
5060 detail: append_capability_problem_detail(
5061 status.health.detail,
5062 capability_detail,
5063 ),
5064 metrics: status.health.metrics,
5065 consecutive_failures: status.health.consecutive_failures,
5066 late_answer_count: status.health.late_answer_count,
5067 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5068 last_action: status.health.last_action,
5069 last_action_ms: status.health.last_action_ms,
5070 last_probe_ms: status.health.last_probe_ms,
5071 })
5072 })
5073 .collect::<Result<Vec<_>, RouterError>>()?;
5074 let response = ClientControlResponse::SupervisorHealth {
5075 generation,
5076 modules,
5077 };
5078 Ok(vec![control_response_body_frame(
5079 &frame,
5080 &response,
5081 "ClientControlResponse::SupervisorHealth",
5082 )?])
5083 }
5084
5085 async fn handle_supervisor_restart(
5086 &self,
5087 frame: Frame,
5088 module_id: String,
5089 drain_timeout_ms: Option<u64>,
5090 requester: Principal,
5091 ) -> Result<Vec<Frame>, RouterError> {
5092 let operation_lock = self.supervisor.operation_lock();
5093 let _operation_guard = operation_lock.lock().await;
5094 let Some(module) = self.supervisor.get(&module_id) else {
5095 return Ok(vec![control_error_frame(
5096 &frame,
5097 "unknown_module",
5098 format!("module_id '{module_id}' is not supervised"),
5099 )?]);
5100 };
5101
5102 self.route_outages.mark_operator_action(&module_id);
5103 if let Err(err) = module
5104 .restart_requested(drain_timeout_ms, Some(requester))
5105 .await
5106 {
5107 self.route_outages
5108 .operator_action_ended_unrefused(&module_id);
5109 let (code, message) = match err {
5110 crate::supervise::SuperviseError::Disabled { .. } => {
5111 ("module_disabled", err.to_string())
5112 }
5113 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5114 ("swap_in_progress", err.to_string())
5115 }
5116 _ => (
5117 "target_unavailable",
5118 format!("failed to restart module_id '{module_id}': {err}"),
5119 ),
5120 };
5121 return Ok(vec![control_error_frame(&frame, code, message)?]);
5122 }
5123
5124 let response = ClientControlResponse::SupervisorAck {
5125 module_id,
5126 applied: true,
5127 };
5128 Ok(vec![control_response_body_frame(
5129 &frame,
5130 &response,
5131 "ClientControlResponse::SupervisorAck",
5132 )?])
5133 }
5134
5135 async fn handle_supervisor_swap(
5139 &self,
5140 frame: Frame,
5141 module_id: String,
5142 ready_timeout_ms: Option<u64>,
5143 requester: Principal,
5144 ) -> Result<Vec<Frame>, RouterError> {
5145 let module = {
5152 let operation_lock = self.supervisor.operation_lock();
5153 let _operation_guard = operation_lock.lock().await;
5154 self.supervisor.get(&module_id)
5155 };
5156 let Some(module) = module else {
5157 return Ok(vec![control_error_frame(
5158 &frame,
5159 "unknown_module",
5160 format!("module_id '{module_id}' is not supervised"),
5161 )?]);
5162 };
5163
5164 self.route_outages.mark_operator_action(&module_id);
5165 if let Err(err) = module
5166 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5167 .await
5168 {
5169 self.route_outages
5170 .operator_action_ended_unrefused(&module_id);
5171 use crate::supervise::SuperviseError;
5172 let message = err.to_string();
5173 let error = match err {
5174 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5175 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5176 code: "swap_refused".to_string(),
5177 message,
5178 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5179 },
5180 SuperviseError::SwapFailed {
5181 arm,
5182 candidate_exit,
5183 ..
5184 } => ErrorBody {
5185 code: "swap_failed".to_string(),
5186 message,
5187 detail: Some(serde_json::json!({
5188 "arm": arm.as_str(),
5189 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5190 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5191 })),
5192 },
5193 _ => ErrorBody::new(
5194 "target_unavailable",
5195 format!("failed to swap module_id '{module_id}': {message}"),
5196 ),
5197 };
5198 return Ok(vec![control_error_body_frame(&frame, error)?]);
5199 }
5200 self.route_outages
5204 .operator_action_ended_unrefused(&module_id);
5205
5206 let response = ClientControlResponse::SupervisorAck {
5207 module_id,
5208 applied: true,
5209 };
5210 Ok(vec![control_response_body_frame(
5211 &frame,
5212 &response,
5213 "ClientControlResponse::SupervisorAck",
5214 )?])
5215 }
5216
5217 async fn handle_supervisor_reload(
5218 &self,
5219 frame: Frame,
5220 module_id: String,
5221 requester: Principal,
5222 ) -> Result<Vec<Frame>, RouterError> {
5223 let operation_lock = self.supervisor.operation_lock();
5224 let _operation_guard = operation_lock.lock().await;
5225 let Some(module) = self.supervisor.get(&module_id) else {
5226 return Ok(vec![control_error_frame(
5227 &frame,
5228 "unknown_module",
5229 format!("module_id '{module_id}' is not supervised"),
5230 )?]);
5231 };
5232
5233 self.route_outages.mark_operator_action(&module_id);
5234 if let Err(err) = module.reload_requested(Some(requester)).await {
5235 self.route_outages
5236 .operator_action_ended_unrefused(&module_id);
5237 let (code, message) = match err {
5238 crate::supervise::SuperviseError::Disabled { .. } => {
5239 ("module_disabled", err.to_string())
5240 }
5241 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5242 ("swap_in_progress", err.to_string())
5243 }
5244 _ => (
5245 "reload_failed",
5246 format!("failed to reload module_id '{module_id}': {err}"),
5247 ),
5248 };
5249 return Ok(vec![control_error_frame(&frame, code, message)?]);
5250 }
5251
5252 let response = ClientControlResponse::SupervisorAck {
5253 module_id,
5254 applied: true,
5255 };
5256 Ok(vec![control_response_body_frame(
5257 &frame,
5258 &response,
5259 "ClientControlResponse::SupervisorAck",
5260 )?])
5261 }
5262
5263 async fn handle_supervisor_rescan(
5264 &self,
5265 frame: Frame,
5266 preview: bool,
5267 requester: Principal,
5268 ) -> Result<Vec<Frame>, RouterError> {
5269 let Some(context) = self.rescan.clone() else {
5270 return Ok(vec![control_error_frame(
5271 &frame,
5272 "rescan_unavailable",
5273 "the daemon was not started with a reloadable config path".to_string(),
5274 )?]);
5275 };
5276
5277 let operation_lock = self.supervisor.operation_lock();
5278 let _operation_guard = operation_lock.lock().await;
5279 let loaded = match crate::daemon_config::load_with_activity(&context.config_path) {
5280 Ok(config) => config,
5281 Err(err) => {
5282 return Ok(vec![control_error_frame(
5283 &frame,
5284 "invalid_daemon_config",
5285 format!("supervisor rescan rejected daemon config: {err}"),
5286 )?])
5287 }
5288 };
5289 let Some((config, activity_config)) = loaded else {
5297 return Ok(vec![control_error_frame(
5298 &frame,
5299 "invalid_daemon_config",
5300 format!(
5301 "daemon config not found at {}; refusing to rescan (an absent config would \
5302 retire every supervised module)",
5303 context.config_path.display()
5304 ),
5305 )?]);
5306 };
5307 let (
5308 configured_port,
5309 storage_config,
5310 admission_facts_carrier_module_id,
5311 admission_facts_targets,
5312 scope_authority_owners,
5313 modules,
5314 reserved_capabilities,
5315 ) = (
5316 config.port,
5317 config.storage,
5318 config.admission_facts_carrier_module_id,
5319 config.admission_facts_targets,
5320 config.scope_authority_owners,
5321 config.modules,
5322 config.reserved_capabilities,
5323 );
5324
5325 let mut restart_required = Vec::new();
5334 for section in RestartRequiredSection::ALL {
5335 let changed = match section {
5336 RestartRequiredSection::Port => configured_port != context.configured_port,
5337 RestartRequiredSection::Storage => storage_config != context.storage_config,
5338 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5339 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5340 }
5341 RestartRequiredSection::AdmissionFactsTargets => {
5342 admission_facts_targets != context.admission_facts_targets
5343 }
5344 RestartRequiredSection::ScopeAuthorityOwners => {
5345 scope_authority_owners != context.scope_authority_owners
5346 }
5347 };
5348 if changed {
5349 restart_required.push(section.label().to_string());
5350 }
5351 }
5352 if !restart_required.is_empty() {
5353 warn!(
5354 config_path = %context.config_path.display(),
5355 sections = %restart_required.join(", "),
5356 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5357 );
5358 }
5359
5360 for configured in &modules {
5361 if let Err(err) = validate_spec(&configured.module_spec()) {
5362 return Ok(vec![control_error_frame(
5363 &frame,
5364 "invalid_daemon_config",
5365 format!("supervisor rescan rejected daemon config: {err}"),
5366 )?]);
5367 }
5368 }
5369
5370 let configured_capabilities = modules
5371 .iter()
5372 .map(|module| (module.module_id.clone(), module.enabled))
5373 .collect::<Vec<_>>();
5374 let preview_capability_warnings = if preview {
5375 let (_, registrations) = self.runtime_capability_snapshot()?;
5376 let current_modules = self
5377 .supervisor
5378 .list()
5379 .into_iter()
5380 .map(|module| module.module_id().to_string())
5381 .collect::<BTreeSet<_>>();
5382 let resulting_modules = configured_capabilities.clone();
5383 let removed = current_modules
5384 .into_iter()
5385 .filter(|module_id| {
5386 !resulting_modules
5387 .iter()
5388 .any(|(configured_id, _)| configured_id == module_id)
5389 })
5390 .collect::<Vec<_>>();
5391 self.capability_evaluator.preview_removal_warnings(
5392 resulting_modules,
5393 &removed,
5394 ®istrations,
5395 )
5396 } else {
5397 Vec::new()
5398 };
5399 let result = match self
5400 .reconcile_supervised_modules(&context.supervisor, modules, preview, requester.clone())
5401 .await
5402 {
5403 Ok(result) => result,
5404 Err(message) => {
5405 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5406 }
5407 };
5408 if !preview {
5409 if let Some(ledger) = &self.activity {
5410 ledger.module_rescanned(&result.added, &result.removed, requester);
5411 }
5412 if let Err(code) = self.configure_activity(activity_config).await {
5413 warn!(
5414 code,
5415 "activity retention could not be applied during rescan"
5416 );
5417 }
5418 self.capability_evaluator
5419 .configure(configured_capabilities, reserved_capabilities);
5420 self.capability_evaluator.wake_deadline_loop();
5421 self.refresh_capability_requirements();
5422 }
5423 let mut result = result;
5424 result.restart_required = restart_required;
5425 result.capability_warnings = preview_capability_warnings;
5426 let response = ClientControlResponse::SupervisorRescan { result };
5427 Ok(vec![control_response_body_frame(
5428 &frame,
5429 &response,
5430 "ClientControlResponse::SupervisorRescan",
5431 )?])
5432 }
5433
5434 async fn handle_supervisor_release_reserved(
5435 &self,
5436 frame: Frame,
5437 module_id: String,
5438 ) -> Result<Vec<Frame>, RouterError> {
5439 let Some(context) = self.rescan.clone() else {
5440 return Ok(vec![control_error_frame(
5441 &frame,
5442 "release_unavailable",
5443 "reserved-id release requires a daemon started with a reloadable config path",
5444 )?]);
5445 };
5446 let operation_lock = self.supervisor.operation_lock();
5447 let _operation_guard = operation_lock.lock().await;
5448 let loaded = match crate::daemon_config::load(&context.config_path) {
5449 Ok(Some(config)) => config,
5450 Ok(None) => {
5451 return Ok(vec![control_error_frame(
5452 &frame,
5453 "invalid_daemon_config",
5454 format!(
5455 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5456 context.config_path.display()
5457 ),
5458 )?])
5459 }
5460 Err(err) => {
5461 return Ok(vec![control_error_frame(
5462 &frame,
5463 "invalid_daemon_config",
5464 format!("unable to verify reserved-id release against daemon config: {err}"),
5465 )?])
5466 }
5467 };
5468 if loaded
5469 .modules
5470 .iter()
5471 .any(|configured| configured.module_id == module_id)
5472 {
5473 return Ok(vec![control_error_frame(
5474 &frame,
5475 "reserved_module_configured",
5476 format!(
5477 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5478 ),
5479 )?]);
5480 }
5481 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5482 return Ok(vec![control_error_frame(
5483 &frame,
5484 "reserved_gate_not_retained",
5485 format!(
5486 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5487 ),
5488 )?]);
5489 }
5490
5491 let response = ClientControlResponse::SupervisorAck {
5492 module_id,
5493 applied: true,
5494 };
5495 Ok(vec![control_response_body_frame(
5496 &frame,
5497 &response,
5498 "ClientControlResponse::SupervisorAck",
5499 )?])
5500 }
5501
5502 async fn reconcile_supervised_modules(
5511 &self,
5512 supervisor: &Supervisor,
5513 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5514 preview: bool,
5515 requester: Principal,
5516 ) -> Result<SupervisorRescanResult, String> {
5517 let mut current = BTreeMap::new();
5518 for module in self.supervisor.list() {
5519 let (spec, health) = module.configuration().map_err(|err| {
5520 format!(
5521 "failed to read configuration for module_id '{}': {err}",
5522 module.module_id()
5523 )
5524 })?;
5525 let enabled = module
5526 .status()
5527 .map_err(|err| {
5528 format!(
5529 "failed to read status for module_id '{}': {err}",
5530 module.module_id()
5531 )
5532 })?
5533 .enabled;
5534 current.insert(
5535 module.module_id().to_string(),
5536 (module, spec, health, enabled),
5537 );
5538 }
5539 let configured = configured_modules
5540 .into_iter()
5541 .map(|module| (module.module_id.clone(), module))
5542 .collect::<BTreeMap<_, _>>();
5543
5544 let added = configured
5545 .keys()
5546 .filter(|module_id| !current.contains_key(*module_id))
5547 .cloned()
5548 .collect::<Vec<_>>();
5549 let removed = current
5550 .keys()
5551 .filter(|module_id| !configured.contains_key(*module_id))
5552 .cloned()
5553 .collect::<Vec<_>>();
5554 let mut changed_pending_reload = Vec::new();
5555 let mut configuration_changes = BTreeSet::new();
5556 let mut enabled_changes = BTreeSet::new();
5557 let mut unchanged = 0_u32;
5558
5559 for (module_id, configured_module) in &configured {
5560 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5561 else {
5562 continue;
5563 };
5564 let launch_changed = *current_spec != configured_module.module_spec();
5568 let configuration_changed =
5569 launch_changed || *current_health != configured_module.health;
5570 let enabled_changed = *current_enabled != configured_module.enabled;
5571 if configuration_changed {
5572 configuration_changes.insert(module_id.clone());
5573 }
5574 if launch_changed {
5575 changed_pending_reload.push(module_id.clone());
5576 }
5577 if enabled_changed {
5578 enabled_changes.insert(module_id.clone());
5579 }
5580 if !configuration_changed && !enabled_changed {
5581 unchanged = unchanged.saturating_add(1);
5582 }
5583 }
5584
5585 if preview {
5591 return Ok(SupervisorRescanResult {
5592 added,
5593 removed,
5594 changed_pending_reload,
5595 enabled_changes: enabled_changes.iter().cloned().collect(),
5596 unchanged,
5597 preview: true,
5598 restart_required: Vec::new(),
5602 capability_warnings: Vec::new(),
5603 });
5604 }
5605
5606 for module_id in &removed {
5607 let module = ¤t
5608 .get(module_id)
5609 .expect("removed module came from current supervisor state")
5610 .0;
5611 module
5612 .retire_requested(Some(requester.clone()))
5613 .await
5614 .map_err(|err| {
5615 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5616 })?;
5617 self.supervisor.record_rescan_removal(module_id);
5651 self.supervisor.retire(module_id);
5652 self.route_outages.forget(module_id);
5653 }
5654
5655 for module_id in configured.keys() {
5656 let Some((module, _, _, _)) = current.get(module_id) else {
5657 continue;
5658 };
5659 let configured_module = configured
5660 .get(module_id)
5661 .expect("configured module id came from configured map");
5662 if configuration_changes.contains(module_id) {
5663 module
5664 .update_configuration(
5665 configured_module.module_spec(),
5666 configured_module.health.clone(),
5667 configured_module.drain_timeout_ms,
5668 )
5669 .await
5670 .map_err(|err| {
5671 format!(
5672 "failed to update module_id '{module_id}' configuration during rescan: {err}"
5673 )
5674 })?;
5675 }
5676 if enabled_changes.contains(module_id) {
5677 self.route_outages.mark_operator_action(module_id);
5680 module
5681 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
5682 .await
5683 .map_err(|err| {
5684 self.route_outages.operator_action_ended_unrefused(module_id);
5685 format!(
5686 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5687 configured_module.enabled
5688 )
5689 })?;
5690 }
5691 }
5692
5693 for module_id in &added {
5694 let configured_module = configured
5695 .get(module_id)
5696 .expect("added module id came from configured map");
5697 supervisor
5698 .supervise_configured_requested(
5699 configured_module.module_spec(),
5700 configured_module.enabled,
5701 configured_module.health.clone(),
5702 configured_module.drain_timeout_ms,
5703 configured_module.restart,
5704 Some(requester.clone()),
5705 )
5706 .map_err(|err| {
5707 format!("failed to add module_id '{module_id}' during rescan: {err}")
5708 })?;
5709 }
5710
5711 Ok(SupervisorRescanResult {
5712 added,
5713 removed,
5714 changed_pending_reload,
5715 enabled_changes: enabled_changes.iter().cloned().collect(),
5716 unchanged,
5717 preview: false,
5718 restart_required: Vec::new(),
5721 capability_warnings: Vec::new(),
5722 })
5723 }
5724
5725 async fn handle_supervisor_set_enabled(
5726 &self,
5727 frame: Frame,
5728 module_id: String,
5729 enabled: bool,
5730 requester: Principal,
5731 ) -> Result<Vec<Frame>, RouterError> {
5732 let operation_lock = self.supervisor.operation_lock();
5733 let _operation_guard = operation_lock.lock().await;
5734 let Some(module) = self.supervisor.get(&module_id) else {
5735 return Ok(vec![control_error_frame(
5736 &frame,
5737 "unknown_module",
5738 format!("module_id '{module_id}' is not supervised"),
5739 )?]);
5740 };
5741
5742 self.route_outages.mark_operator_action(&module_id);
5745 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
5746 Ok(applied) => applied,
5747 Err(err) => {
5748 self.route_outages
5749 .operator_action_ended_unrefused(&module_id);
5750 return Ok(vec![control_error_frame(
5751 &frame,
5752 "target_unavailable",
5753 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
5754 )?]);
5755 }
5756 };
5757 if !applied {
5758 self.route_outages
5761 .operator_action_ended_unrefused(&module_id);
5762 }
5763
5764 self.capability_evaluator.wake_deadline_loop();
5765 self.refresh_capability_requirements();
5766 let response = ClientControlResponse::SupervisorAck { module_id, applied };
5767 Ok(vec![control_response_body_frame(
5768 &frame,
5769 &response,
5770 "ClientControlResponse::SupervisorAck",
5771 )?])
5772 }
5773
5774 async fn handle_supervisor_health_probe(
5775 &self,
5776 frame: Frame,
5777 module_id: String,
5778 ) -> Result<Vec<Frame>, RouterError> {
5779 self.refresh_capability_requirements();
5780 let Some(registration) = self
5781 .registry
5782 .get_module(&module_id)
5783 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5784 else {
5785 return Ok(vec![control_error_frame(
5786 &frame,
5787 "unknown_module",
5788 format!("module_id '{module_id}' is not registered"),
5789 )?]);
5790 };
5791
5792 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
5805 {
5806 return Ok(vec![control_error_frame(
5807 &frame,
5808 "health_not_advertised",
5809 format!("module_id '{module_id}' did not advertise health.check"),
5810 )?]);
5811 }
5812
5813 let deadline = Instant::now() + self.health_probe_timeout;
5814 let pending = match self.forwarding.begin_module_control_rpc_for(
5815 &module_id,
5816 MODULE_CONTROL_OP_HEALTH_CHECK,
5817 deadline,
5818 ) {
5819 Ok(pending) => pending,
5820 Err(err) => {
5821 return Ok(vec![control_error_frame(
5822 &frame,
5823 forwarding_error_code(&err),
5824 err.to_string(),
5825 )?])
5826 }
5827 };
5828
5829 let PendingModuleControlRpc {
5830 endpoint,
5831 module_sink,
5832 negotiated_ver,
5833 corr: probe_corr,
5834 receiver,
5835 } = pending;
5836 let mut guard =
5837 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
5838 let probe_body =
5839 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
5840 RouterError::backend(
5841 0,
5842 frame.header.corr,
5843 format!("failed to encode health.check request: {err}"),
5844 )
5845 })?;
5846 let probe_frame = Frame::build_with_version(
5847 negotiated_ver,
5848 FrameType::Request,
5849 control_flags(),
5850 0,
5851 0,
5852 probe_corr,
5853 probe_body,
5854 )
5855 .map_err(RouterError::FrameBuild)?;
5856
5857 if let Err(err) = module_sink.send(probe_frame).await {
5858 return Ok(vec![control_error_frame(
5859 &frame,
5860 "target_unavailable",
5861 err.to_string(),
5862 )?]);
5863 }
5864
5865 match timeout_at(deadline, receiver).await {
5866 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
5867 guard.disarm();
5868 let Some(report) = response.health_report() else {
5869 return Ok(vec![control_error_frame(
5870 &frame,
5871 "invalid_control_body",
5872 "health.check RPC returned a non-health response",
5873 )?]);
5874 };
5875 let HealthReport {
5880 status,
5881 detail,
5882 metrics,
5883 } = report;
5884 let capability_detail = self
5885 .capability_evaluator
5886 .required_problem_detail(&module_id);
5887 let response = ClientControlResponse::SupervisorHealthProbe {
5888 module_id,
5889 status,
5890 detail: append_capability_problem_detail(detail, capability_detail),
5891 metrics,
5892 };
5893 Ok(vec![control_response_body_frame(
5894 &frame,
5895 &response,
5896 "ClientControlResponse::SupervisorHealthProbe",
5897 )?])
5898 }
5899 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
5900 guard.disarm();
5901 Ok(vec![control_error_body_frame(&frame, body)?])
5902 }
5903 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
5904 guard.disarm();
5905 Ok(vec![control_error_frame(
5906 &frame,
5907 "target_unavailable",
5908 message,
5909 )?])
5910 }
5911 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
5912 guard.disarm();
5913 Ok(vec![control_error_frame(
5914 &frame,
5915 "invalid_control_body",
5916 message,
5917 )?])
5918 }
5919 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
5920 guard.disarm();
5921 Ok(vec![control_error_frame(
5922 &frame,
5923 "invalid_control_body",
5924 format!("expected module-control op '{expected}', got '{actual}'"),
5925 )?])
5926 }
5927 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
5928 guard.disarm();
5929 Ok(vec![control_error_frame(
5930 &frame,
5931 "module_timeout",
5932 format!(
5933 "module_id '{module_id}' answered health.check after {:?}",
5934 self.health_probe_timeout
5935 ),
5936 )?])
5937 }
5938 Ok(Err(_)) => Ok(vec![control_error_frame(
5939 &frame,
5940 "target_unavailable",
5941 "health.check waiter was canceled before the module responded",
5942 )?]),
5943 Err(_) => Ok(vec![control_error_frame(
5944 &frame,
5945 "module_timeout",
5946 format!(
5947 "module_id '{module_id}' did not answer health.check within {:?}",
5948 self.health_probe_timeout
5949 ),
5950 )?]),
5951 }
5952 }
5953
5954 fn supervisor_status(
5955 &self,
5956 module_id: &str,
5957 corr: u64,
5958 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
5959 self.supervisor
5960 .get(module_id)
5961 .map(|module| {
5962 let warming = module.is_warming_for_control("status").map_err(|err| {
5963 RouterError::backend(
5964 0,
5965 corr,
5966 format!(
5967 "failed to read supervisor warming state for module_id '{module_id}': {err}"
5968 ),
5969 )
5970 })?;
5971 module.status_for_control("status").map_err(|err| {
5972 RouterError::backend(
5973 0,
5974 corr,
5975 format!(
5976 "failed to read supervisor status for module_id '{module_id}': {err}"
5977 ),
5978 )
5979 }).map(|status| (status, warming))
5980 })
5981 .transpose()
5982 }
5983
5984 fn guard_module_control_op(
5985 &self,
5986 frame: &Frame,
5987 module_id: &str,
5988 op: &str,
5989 ) -> Result<Option<Frame>, RouterError> {
5990 if self.module_grants_op(module_id, op, frame.header.corr)? {
5991 return Ok(None);
5992 }
5993
5994 Ok(Some(control_error_frame(
5995 frame,
5996 "op_not_allowed",
5997 format!("module_id '{module_id}' did not grant control op '{op}'"),
5998 )?))
5999 }
6000
6001 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6002 let Some(registration) = self
6003 .registry
6004 .get_module(module_id)
6005 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6006 else {
6007 return Ok(false);
6008 };
6009 Ok(module_registration_grants_op(®istration.control_ops, op))
6010 }
6011
6012 fn handle_status_update(
6013 &self,
6014 endpoint: ModuleEndpointId,
6015 frame: Frame,
6016 ) -> Result<Vec<Frame>, RouterError> {
6017 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6018 Ok(update) => update,
6019 Err(err) => {
6020 if is_known_module_push_op(&frame.body) {
6025 return Ok(vec![control_error_frame(
6026 &frame,
6027 "invalid_control_body",
6028 format!("malformed module control push body: {err}"),
6029 )?]);
6030 }
6031 return Ok(Vec::new());
6032 }
6033 };
6034
6035 match update {
6036 ModuleControlPush::RouteStatus {
6037 route_channel,
6038 route_epoch,
6039 status,
6040 } => {
6041 self.forwarding
6042 .cache_status(endpoint, route_channel, route_epoch, status)
6043 .map_err(RouterError::Forwarding)?;
6044 }
6045 }
6046 Ok(Vec::new())
6047 }
6048
6049 fn handle_route_poll(
6050 &self,
6051 ctx: &RouteCtx,
6052 frame: Frame,
6053 route_channel: u16,
6054 route_epoch: u32,
6055 kind: PollKind,
6056 ) -> Result<Vec<Frame>, RouterError> {
6057 let snapshot = self
6058 .forwarding
6059 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6060 .map_err(RouterError::Forwarding)?;
6061 let response = match (kind, snapshot) {
6062 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6063 ClientControlResponse::RoutePoll {
6064 route_channel,
6065 route_epoch,
6066 status,
6067 live: None,
6068 }
6069 }
6070 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6071 route_channel,
6072 route_epoch,
6073 status: None,
6074 live: None,
6075 },
6076 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6077 let live = self
6103 .process_liveness
6104 .as_ref()
6105 .and_then(|source| source.process_live(&module_id))
6106 .unwrap_or(true);
6107 ClientControlResponse::RoutePoll {
6108 route_channel,
6109 route_epoch,
6110 status: None,
6111 live: Some(live),
6112 }
6113 }
6114 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6115 route_channel,
6116 route_epoch,
6117 status: None,
6118 live: Some(false),
6119 },
6120 };
6121
6122 Ok(vec![control_response_body_frame(
6123 &frame,
6124 &response,
6125 "ClientControlResponse::RoutePoll",
6126 )?])
6127 }
6128
6129 pub(crate) fn observe_module_control_completion(
6130 &self,
6131 completion: ModuleControlRpcCompletion,
6132 ) -> bool {
6133 match completion {
6134 ModuleControlRpcCompletion::Unknown => false,
6135 ModuleControlRpcCompletion::Settled => true,
6136 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6137 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6138 info!(
6139 module_id = %module_id,
6140 latency_ms,
6141 "late health.check answer proves the module is alive"
6142 );
6143 match self
6144 .supervisor
6145 .record_late_health_answer(&module_id, latency_ms)
6146 {
6147 Ok(true) => {}
6148 Ok(false) => debug!(
6149 module_id = %module_id,
6150 latency_ms,
6151 "late health.check answer has no active supervisor snapshot"
6152 ),
6153 Err(err) => warn!(
6154 module_id = %module_id,
6155 latency_ms,
6156 error = %err,
6157 "failed to record late health.check answer"
6158 ),
6159 }
6160 true
6161 }
6162 }
6163 }
6164
6165 fn refuse_to_end_module_connection_for_a_client(
6186 &self,
6187 module_connection_id: ConnectionId,
6188 corr: u64,
6189 err: ForwardingError,
6190 ) -> Result<(), RouterError> {
6191 if let ForwardingError::ConnectionClosing { connection_id } = err {
6192 if connection_id != module_connection_id {
6193 warn!(
6194 module_connection_id = module_connection_id.get(),
6195 client_connection_id = connection_id.get(),
6196 corr,
6197 "dropping a route.bind response for a closing client; the module connection keeps serving"
6198 );
6199 return Ok(());
6200 }
6201 }
6202 Err(RouterError::Forwarding(err))
6203 }
6204
6205 fn handle_module_relay_response(
6206 &self,
6207 connection_id: ConnectionId,
6208 frame: Frame,
6209 ) -> Result<Vec<Frame>, RouterError> {
6210 let mut secondary_error = None;
6211 let outcome = match frame.header.ty {
6212 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6213 Ok(probe) if probe.op == "route.bind" => {
6214 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6215 Ok(ModuleControlResponse::RouteBindAck {}) => {
6216 RouteBindRelayOutcome::Accepted
6217 }
6218 Ok(other) => {
6219 let message =
6220 format!("route.bind response carried unexpected body: {other:?}");
6221 secondary_error = Some(control_error_frame(
6222 &frame,
6223 "invalid_control_body",
6224 message.clone(),
6225 )?);
6226 RouteBindRelayOutcome::ModuleGone(message)
6227 }
6228 Err(err) => {
6229 let message = format!("malformed route.bind response body: {err}");
6230 secondary_error = Some(control_error_frame(
6231 &frame,
6232 "invalid_control_body",
6233 message.clone(),
6234 )?);
6235 RouteBindRelayOutcome::ModuleGone(message)
6236 }
6237 }
6238 }
6239 Ok(probe) => {
6240 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6241 {
6242 Ok(response) => ModuleControlRpcOutcome::Response(response),
6243 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6244 "malformed {} response body: {err}",
6245 probe.op
6246 )),
6247 };
6248 let completion = self
6249 .forwarding
6250 .complete_module_control_rpc(
6251 connection_id,
6252 frame.header.corr,
6253 Some(&probe.op),
6254 outcome,
6255 )
6256 .map_err(RouterError::Forwarding)?;
6257 if !self.observe_module_control_completion(completion) {
6258 debug!(
6259 connection_id = connection_id.get(),
6260 corr = frame.header.corr,
6261 op = %probe.op,
6262 "dropping late or unknown module-control RPC response"
6263 );
6264 }
6265 return Ok(Vec::new());
6266 }
6267 Err(err) => {
6268 if let Some(expected_op) = self
6269 .forwarding
6270 .pending_module_control_op(connection_id, frame.header.corr)
6271 .map_err(RouterError::Forwarding)?
6272 {
6273 let completion = self
6274 .forwarding
6275 .complete_module_control_rpc(
6276 connection_id,
6277 frame.header.corr,
6278 None,
6279 ModuleControlRpcOutcome::MalformedResponse(format!(
6280 "malformed {expected_op} response body: {err}"
6281 )),
6282 )
6283 .map_err(RouterError::Forwarding)?;
6284 if !self.observe_module_control_completion(completion) {
6285 debug!(
6286 connection_id = connection_id.get(),
6287 corr = frame.header.corr,
6288 "dropping late malformed module-control RPC response"
6289 );
6290 }
6291 return Ok(Vec::new());
6292 }
6293 let message = format!("malformed route.bind response body: {err}");
6294 secondary_error = Some(control_error_frame(
6295 &frame,
6296 "invalid_control_body",
6297 message.clone(),
6298 )?);
6299 RouteBindRelayOutcome::ModuleGone(message)
6300 }
6301 },
6302 FrameType::Error => {
6303 if self
6304 .forwarding
6305 .pending_module_control_op(connection_id, frame.header.corr)
6306 .map_err(RouterError::Forwarding)?
6307 .is_some()
6308 {
6309 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6310 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6311 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6312 "malformed module-control ERROR body: {err}"
6313 )),
6314 };
6315 let completion = self
6316 .forwarding
6317 .complete_module_control_rpc(
6318 connection_id,
6319 frame.header.corr,
6320 None,
6321 outcome,
6322 )
6323 .map_err(RouterError::Forwarding)?;
6324 if !self.observe_module_control_completion(completion) {
6325 debug!(
6326 connection_id = connection_id.get(),
6327 corr = frame.header.corr,
6328 "dropping late or unknown module-control RPC error"
6329 );
6330 }
6331 return Ok(Vec::new());
6332 }
6333 match serde_json::from_slice::<ErrorBody>(&frame.body) {
6334 Ok(body) => RouteBindRelayOutcome::Rejected(body),
6335 Err(err) => {
6336 let message = format!("malformed route.bind ERROR body: {err}");
6337 secondary_error = Some(control_error_frame(
6338 &frame,
6339 "invalid_control_body",
6340 message.clone(),
6341 )?);
6342 RouteBindRelayOutcome::ModuleGone(message)
6343 }
6344 }
6345 }
6346 ty => {
6347 return Ok(vec![control_error_frame(
6348 &frame,
6349 "unsupported_control_frame",
6350 format!("unsupported module channel-0 frame {ty:?}"),
6351 )?])
6352 }
6353 };
6354
6355 let settled =
6356 self.forwarding
6357 .complete_pending_relay(connection_id, frame.header.corr, outcome);
6358 let completion = match settled {
6359 Ok(completion) => completion,
6360 Err(err) => {
6361 self.refuse_to_end_module_connection_for_a_client(
6362 connection_id,
6363 frame.header.corr,
6364 err,
6365 )?;
6366 return Ok(secondary_error.into_iter().collect());
6367 }
6368 };
6369 if let Some(target) = completion.abandoned.as_ref() {
6370 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6371 }
6372 if !completion.settled {
6373 debug!(
6374 connection_id = connection_id.get(),
6375 corr = frame.header.corr,
6376 frame_type = ?frame.header.ty,
6377 "dropping late or unknown route.bind relay response"
6378 );
6379 }
6380 Ok(secondary_error.into_iter().collect())
6381 }
6382
6383 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6384 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6385 self.cleanup_connection_with_end_reason(
6389 connection_id,
6390 RegistrationEndReason::ExplicitGoodbye,
6391 )
6392 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6393 Ok(Vec::new())
6394 }
6395}
6396
6397impl Default for ControlHandler {
6398 fn default() -> Self {
6399 Self::new(Arc::new(Registry::default()))
6400 }
6401}
6402
6403impl crate::supervise::SwapPromotionObserver for ControlHandler {
6404 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6405 self.apply_registration_capabilities(registration);
6406 }
6407}
6408
6409fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6410 CapabilityRequirementStatus {
6411 consumer: status.consumer,
6412 capability: status.capability,
6413 need: match status.need {
6414 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6415 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6416 },
6417 verdict: status.verdict.as_str().to_string(),
6418 episode_seq: status.episode_seq,
6419 config_satisfiable: status.config_satisfiable,
6420 runtime_available: status.runtime_available,
6421 detail: status.detail,
6422 }
6423}
6424
6425fn append_capability_problem_detail(
6426 detail: Option<String>,
6427 capability_detail: Option<String>,
6428) -> Option<String> {
6429 match (detail, capability_detail) {
6430 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6431 (Some(detail), None) => Some(detail),
6432 (None, Some(capability_detail)) => Some(capability_detail),
6433 (None, None) => None,
6434 }
6435}
6436
6437fn subc_ops() -> Vec<String> {
6438 SUBC_CONTROL_OPS
6439 .iter()
6440 .map(|op| (*op).to_string())
6441 .collect()
6442}
6443
6444fn module_subc_ops() -> Vec<String> {
6445 SUBC_CONTROL_OPS
6446 .iter()
6447 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6448 .map(|op| (*op).to_string())
6449 .collect()
6450}
6451
6452#[cfg(test)]
6453fn module_baseline_control_ops() -> Vec<String> {
6454 MODULE_BASELINE_CONTROL_OPS
6455 .iter()
6456 .map(|op| (*op).to_string())
6457 .collect()
6458}
6459
6460fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6461 let mut seen = HashSet::new();
6462 let mut effective = Vec::new();
6463 for op in MODULE_BASELINE_CONTROL_OPS {
6464 if seen.insert((*op).to_string()) {
6465 effective.push((*op).to_string());
6466 }
6467 }
6468 for op in declared.unwrap_or_default() {
6469 if seen.insert(op.clone()) {
6470 effective.push(op);
6471 }
6472 }
6473 effective
6474}
6475
6476fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6477 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6478}
6479
6480fn target_module_id(target: &RouteTarget) -> &str {
6481 match target {
6482 RouteTarget::ToolProvider { module_id }
6483 | RouteTarget::ManagementSurface { module_id }
6484 | RouteTarget::InternalService { module_id, .. } => module_id,
6485 }
6486}
6487
6488fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6489 roles.iter().any(|role| match (target, role) {
6490 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6491 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6492 (
6493 RouteTarget::InternalService { service_id, .. },
6494 ProviderRole::InternalService {
6495 service_id: provided,
6496 ..
6497 },
6498 ) => service_id == provided,
6499 _ => false,
6500 })
6501}
6502
6503fn is_routable_role(role: &ProviderRole) -> bool {
6504 matches!(
6505 role,
6506 ProviderRole::ToolProvider { .. }
6507 | ProviderRole::ManagementSurface { .. }
6508 | ProviderRole::InternalService { .. }
6509 )
6510}
6511
6512#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6513enum ControlRequestBodyError {
6514 UnknownOp,
6515 InvalidBody,
6516}
6517
6518#[derive(Debug, Deserialize)]
6519struct ControlOpProbe {
6520 op: String,
6521}
6522
6523const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6526
6527fn is_known_module_push_op(body: &[u8]) -> bool {
6528 serde_json::from_slice::<ControlOpProbe>(body)
6529 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6530 .unwrap_or(false)
6531}
6532
6533fn is_known_module_request_op(body: &[u8]) -> bool {
6534 serde_json::from_slice::<ControlOpProbe>(body)
6535 .map(|probe| is_module_to_subc_op(&probe.op))
6536 .unwrap_or(false)
6537}
6538
6539fn is_module_to_subc_op(op: &str) -> bool {
6540 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6541}
6542
6543fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6544 debug!(
6545 op = %op,
6546 connection_id = connection_id.get(),
6547 corr,
6548 "control dispatch"
6549 );
6550}
6551
6552fn log_slow_control_dispatch(
6553 dispatch_started_at: Option<StdInstant>,
6554 op: &'static str,
6555 connection_id: ConnectionId,
6556 corr: u64,
6557) {
6558 let Some(dispatch_started_at) = dispatch_started_at else {
6559 return;
6560 };
6561 let elapsed = dispatch_started_at.elapsed();
6562 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6563 warn!(
6564 op = %op,
6565 connection_id = connection_id.get(),
6566 corr,
6567 elapsed_ms = elapsed.as_millis() as u64,
6568 "slow control dispatch"
6569 );
6570 }
6571}
6572
6573fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6574 match request {
6575 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6576 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6577 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6578 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6579 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6580 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6581 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6582 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6583 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6584 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6585 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6586 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6587 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6588 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6589 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6590 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6591 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6592 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6593 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6594 }
6595}
6596
6597fn principal_label(principal: &Principal) -> String {
6598 match principal {
6599 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6600 Principal::Direct => "direct".to_string(),
6601 other => format!("{other:?}"),
6602 }
6603}
6604
6605fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6606 match request {
6607 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6608 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6609 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6610 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6611 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6612 }
6613}
6614
6615fn parse_client_control_request(
6616 body: &[u8],
6617) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6618 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6619 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6620 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6621 ControlRequestBodyError::InvalidBody
6622 }
6623 Ok(_) => ControlRequestBodyError::UnknownOp,
6624 Err(_) => ControlRequestBodyError::InvalidBody,
6625 };
6626 (err, classification)
6627 })
6628}
6629
6630fn parse_module_control_request_from_module(
6631 body: &[u8],
6632) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6633 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6634 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6635 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6636 Ok(_) => ControlRequestBodyError::UnknownOp,
6637 Err(_) => ControlRequestBodyError::InvalidBody,
6638 };
6639 (err, classification)
6640 })
6641}
6642
6643#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6644enum ProviderRoleKind {
6645 ToolProvider,
6646 PipelineStage,
6647 ManagementSurface,
6648 InternalService,
6649}
6650
6651fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6652 match role {
6653 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6654 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6655 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6656 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
6657 }
6658}
6659
6660fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
6661 roles.iter().map(provider_role_kind).collect()
6662}
6663
6664const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
6667
6668#[derive(Debug, Default, PartialEq, Eq)]
6670struct ScopeOutcomeCounts {
6671 created: usize,
6672 replaced: usize,
6673 updated: usize,
6674 unchanged: usize,
6675 refused: usize,
6676}
6677
6678impl ScopeOutcomeCounts {
6679 fn of(results: &[ScopeRecordResult]) -> Self {
6680 let mut counts = Self::default();
6681 for result in results {
6682 let slot = match result.outcome {
6683 ScopeRecordOutcome::Created => &mut counts.created,
6684 ScopeRecordOutcome::Replaced => &mut counts.replaced,
6685 ScopeRecordOutcome::Updated => &mut counts.updated,
6686 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
6687 ScopeRecordOutcome::Refused => &mut counts.refused,
6688 };
6689 *slot += 1;
6690 }
6691 counts
6692 }
6693}
6694
6695#[cfg(test)]
6696mod scope_outcome_count_tests {
6697 use super::*;
6698
6699 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
6700 ScopeRecordResult {
6701 scope_ref: "r".to_string(),
6702 scope_epoch: 1,
6703 outcome,
6704 code: None,
6705 message: None,
6706 version: None,
6707 parent_state: None,
6708 }
6709 }
6710
6711 #[test]
6714 fn every_outcome_is_counted_in_its_own_field() {
6715 let results = [
6716 result(ScopeRecordOutcome::Created),
6717 result(ScopeRecordOutcome::Created),
6718 result(ScopeRecordOutcome::Replaced),
6719 result(ScopeRecordOutcome::Updated),
6720 result(ScopeRecordOutcome::Unchanged),
6721 result(ScopeRecordOutcome::Refused),
6722 result(ScopeRecordOutcome::Refused),
6723 result(ScopeRecordOutcome::Refused),
6724 ];
6725 assert_eq!(
6726 ScopeOutcomeCounts::of(&results),
6727 ScopeOutcomeCounts {
6728 created: 2,
6729 replaced: 1,
6730 updated: 1,
6731 unchanged: 1,
6732 refused: 3,
6733 }
6734 );
6735 }
6736}
6737
6738fn capability_census_trigger(
6742 old: Option<&CapabilityDeclarations>,
6743 new: Option<&CapabilityDeclarations>,
6744) -> bool {
6745 let old_provides = old
6746 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
6747 .unwrap_or_default();
6748 let old_denies = old
6749 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
6750 .unwrap_or_default();
6751 let new = new.cloned().unwrap_or(CapabilityDeclarations {
6752 provides: Vec::new(),
6753 requires: Vec::new(),
6754 must_never_reach: Vec::new(),
6755 });
6756
6757 new.provides
6758 .iter()
6759 .any(|capability| !old_provides.contains(capability))
6760 || new
6761 .must_never_reach
6762 .iter()
6763 .any(|capability| !old_denies.contains(capability))
6764}
6765
6766fn denied_capability<'a>(
6769 opening_manifest: &'a ModuleManifest,
6770 target_manifest: &ModuleManifest,
6771) -> Option<&'a str> {
6772 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
6773 let target_capabilities = target_manifest.capabilities.as_ref()?;
6774 opening_capabilities
6775 .must_never_reach
6776 .iter()
6777 .find(|denied| {
6778 target_capabilities
6779 .provides
6780 .iter()
6781 .any(|provided| provided == *denied)
6782 })
6783 .map(String::as_str)
6784}
6785
6786fn catalog_update_frozen_field_message(
6787 registered: &ModuleManifest,
6788 provides: &[ProviderRole],
6789) -> Option<String> {
6790 let old_has_provides = !registered.provides.is_empty();
6791 let new_has_provides = !provides.is_empty();
6792 if old_has_provides != new_has_provides {
6793 return Some(format!(
6794 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
6795 registered.module_id
6796 ));
6797 }
6798
6799 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
6800 return Some(format!(
6801 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
6802 registered.module_id
6803 ));
6804 }
6805
6806 let registered_concurrency = manifest_concurrency(registered);
6807 let mut candidate = registered.clone();
6808 candidate.provides = provides.to_vec();
6809 let candidate_concurrency = manifest_concurrency(&candidate);
6810 if candidate_concurrency != registered_concurrency {
6811 return Some(format!(
6812 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
6813 registered.module_id, registered_concurrency, candidate_concurrency
6814 ));
6815 }
6816
6817 None
6820}
6821
6822fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
6823 manifest.provides.iter().any(is_routable_role)
6824}
6825
6826fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
6832 manifest
6833 .provides
6834 .iter()
6835 .find_map(|provider| match provider {
6836 ProviderRole::ToolProvider { concurrency, .. }
6837 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
6838 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
6839 })
6840 .unwrap_or(Concurrency::ModuleManaged)
6841}
6842
6843fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
6850 let has_management_surface = manifest
6851 .provides
6852 .iter()
6853 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
6854 if !has_management_surface {
6855 return false;
6856 }
6857 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
6858 return false;
6859 };
6860 let Some(provides) = raw
6861 .get("manifest")
6862 .and_then(|manifest| manifest.get("provides"))
6863 .and_then(serde_json::Value::as_array)
6864 else {
6865 return false;
6866 };
6867 provides.iter().any(|role| {
6872 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
6873 && role.get("concurrency").is_none()
6874 })
6875}
6876
6877fn negotiate_version(peer_version: u8) -> Result<u8, String> {
6878 if peer_version != PROTOCOL_VERSION {
6879 return Err(format!(
6880 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
6881 ));
6882 }
6883 Ok(PROTOCOL_VERSION)
6884}
6885
6886fn pong(frame: &Frame) -> Result<Frame, RouterError> {
6887 Frame::build_with_version(
6888 response_version(frame),
6889 FrameType::Pong,
6890 frame.header.flags,
6891 0,
6892 0,
6893 frame.header.corr,
6894 Vec::new(),
6895 )
6896 .map_err(RouterError::FrameBuild)
6897}
6898
6899pub(crate) fn control_error_frame(
6900 frame: &Frame,
6901 code: &'static str,
6902 message: impl Into<String>,
6903) -> Result<Frame, RouterError> {
6904 control_error_body_frame(
6905 frame,
6906 ErrorBody {
6907 code: code.to_string(),
6908 message: message.into(),
6909 detail: None,
6910 },
6911 )
6912}
6913
6914fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
6915 let body = serde_json::to_vec(&error).map_err(|err| {
6916 RouterError::backend(
6917 0,
6918 frame.header.corr,
6919 format!("failed to encode control ERROR: {err}"),
6920 )
6921 })?;
6922
6923 Frame::build_with_version(
6924 response_version(frame),
6925 FrameType::Error,
6926 control_flags(),
6927 0,
6928 0,
6929 frame.header.corr,
6930 body,
6931 )
6932 .map_err(RouterError::FrameBuild)
6933}
6934
6935fn control_response_body_frame<T: Serialize>(
6936 frame: &Frame,
6937 reply: &T,
6938 label: &'static str,
6939) -> Result<Frame, RouterError> {
6940 let body = serde_json::to_vec(reply).map_err(|err| {
6941 RouterError::backend(
6942 0,
6943 frame.header.corr,
6944 format!("failed to encode {label}: {err}"),
6945 )
6946 })?;
6947
6948 Frame::build_with_version(
6949 response_version(frame),
6950 FrameType::Response,
6951 control_flags(),
6952 0,
6953 0,
6954 frame.header.corr,
6955 body,
6956 )
6957 .map_err(RouterError::FrameBuild)
6958}
6959
6960fn forwarding_error_code(err: &ForwardingError) -> &'static str {
6990 match err {
6991 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
6992 ForwardingError::NoModuleConnection => "target_unavailable",
6993 ForwardingError::ModuleReloading { .. } => "module_reloading",
6994 ForwardingError::ClientRouteChannelExhausted { .. }
6995 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
6996 ForwardingError::StaleModuleEndpoint
6997 | ForwardingError::UnknownReservation { .. }
6998 | ForwardingError::ConnectionClosing { .. }
6999 | ForwardingError::ClientEgressClosed { .. }
7000 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7001 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7004 ForwardingError::RelayCorrelationExhausted
7005 | ForwardingError::RouteOpenBuild(_)
7006 | ForwardingError::Poisoned => "forwarding_error",
7007 }
7008}
7009
7010fn response_version(frame: &Frame) -> u8 {
7011 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7012 frame.header.ver
7013 } else {
7014 PROTOCOL_VERSION
7015 }
7016}
7017
7018fn control_flags() -> Flags {
7019 Flags::new(false, Priority::Passive, false)
7020}
7021
7022fn send_goodbye_target_best_effort(
7027 counters: &DaemonCounters,
7028 target: &GoodbyeTarget,
7029 context: &'static str,
7030) {
7031 let Ok(frame) = Frame::build_with_version(
7032 target.negotiated_ver,
7033 FrameType::Goodbye,
7034 control_flags(),
7035 target.channel,
7036 target.epoch,
7037 0,
7038 Vec::new(),
7039 ) else {
7040 return;
7041 };
7042 crate::forwarding::send_module_route_goodbye(
7043 counters,
7044 &target.sink,
7045 frame,
7046 target.module_id.as_deref(),
7047 context,
7048 );
7049}
7050
7051pub(crate) fn send_route_control_pushes(
7052 forwarding: &ForwardingTable,
7053 routes: Vec<EndpointRoute>,
7054 push: ClientControlPush,
7055) {
7056 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7057 for route in routes {
7058 let target = route.goodbye_target;
7059 if let Some((existing, channels)) = targets
7060 .iter_mut()
7061 .find(|(existing, _)| existing.connection_id == target.connection_id)
7062 {
7063 debug_assert_eq!(
7064 existing.negotiated_ver, target.negotiated_ver,
7065 "one connection cannot negotiate multiple frame versions"
7066 );
7067 if !channels.contains(&target.channel) {
7068 channels.push(target.channel);
7069 }
7070 continue;
7071 }
7072 let channel = target.channel;
7073 targets.push((target, vec![channel]));
7074 }
7075 for (target, mut channels) in targets {
7076 channels.sort_unstable();
7077 let mut push = push.clone();
7078 match &mut push {
7079 ClientControlPush::RouteClosing {
7080 channels: covered, ..
7081 }
7082 | ClientControlPush::RouteClosed {
7083 channels: covered, ..
7084 } => *covered = channels,
7085 }
7086 let body = match serde_json::to_vec(&push) {
7087 Ok(body) => body,
7088 Err(err) => {
7089 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7090 continue;
7091 }
7092 };
7093 let frame = match Frame::build_with_version(
7094 target.negotiated_ver,
7095 FrameType::Push,
7096 control_flags(),
7097 0,
7098 0,
7099 0,
7100 body.clone(),
7101 ) {
7102 Ok(frame) => frame,
7103 Err(err) => {
7104 warn!(
7105 route_channel = target.channel,
7106 error = %err,
7107 "failed to build route lifecycle control PUSH frame"
7108 );
7109 continue;
7110 }
7111 };
7112 if let Err(err) = target.sink.try_send(frame) {
7113 if target.close_on_delivery_failure() {
7114 warn!(
7115 target_connection_id = target.connection_id.get(),
7116 route_channel = target.channel,
7117 error = %err,
7118 "route lifecycle control PUSH was not delivered to client; closing target connection"
7119 );
7120 let _ = forwarding.escalate_client_delivery_failure(
7121 target.connection_id,
7122 target.channel,
7123 target.epoch,
7124 CloseReason::new(
7125 "route_lifecycle_push_delivery_failed",
7126 format!(
7127 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7128 target.channel
7129 ),
7130 ),
7131 crate::forwarding::UndeliveredFrame {
7132 module_id: target.module_id.as_deref(),
7133 sink: &target.sink,
7134 },
7135 );
7136 }
7137 }
7138 }
7139}
7140
7141#[cfg(test)]
7142mod tests {
7143 mod activity_read {
7144 include!("activity_read_tests.rs");
7145 }
7146
7147 mod lifecycle_activity {
7148 include!("control_lifecycle_tests.rs");
7149 }
7150 #[cfg(unix)]
7151 #[tokio::test]
7152 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7153 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7154 let path = dir.join("subc.jsonc");
7155 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7156 "program":"/bin/sleep","args":["60"],"protocol":"none",
7157 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7158 }}}).to_string()).unwrap();
7159 let mut configured = crate::daemon_config::load(&path)
7160 .unwrap()
7161 .unwrap()
7162 .modules
7163 .pop()
7164 .unwrap();
7165 let registry = std::sync::Arc::new(crate::Registry::default());
7166 let handle = crate::SupervisorHandle::new();
7167 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7168 .with_handle(handle.clone());
7169 let module = supervisor
7170 .supervise_configured_with_health(
7171 configured.module_spec(),
7172 true,
7173 configured.health.clone(),
7174 None,
7175 configured.restart,
7176 )
7177 .unwrap();
7178 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7179 let before = module.status().unwrap().pid;
7180 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7181 configured.health.cadence = std::time::Duration::from_secs(3600);
7182 let health_only = handler
7183 .reconcile_supervised_modules(
7184 &supervisor,
7185 vec![configured.clone()],
7186 false,
7187 Principal::Direct,
7188 )
7189 .await
7190 .unwrap();
7191 assert!(
7192 health_only.changed_pending_reload.is_empty(),
7193 "health policy is already applied live"
7194 );
7195 assert_eq!(module.status().unwrap().pid, before);
7196 assert_eq!(
7197 module.configuration().unwrap().1.http,
7198 configured.health.http
7199 );
7200 configured.args = vec!["61".into()];
7201 let launch = handler
7202 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7203 .await
7204 .unwrap();
7205 assert_eq!(launch.changed_pending_reload, ["stock"]);
7206 assert_eq!(
7207 module.status().unwrap().pid,
7208 before,
7209 "a launch edit is stored until reload"
7210 );
7211 module.drain().await.unwrap();
7212 }
7213 use cortexkit_test_support::ScratchDir;
7214 use std::{
7215 collections::BTreeMap,
7216 fmt,
7217 path::PathBuf,
7218 sync::{Arc, Mutex},
7219 time::Duration,
7220 };
7221
7222 use serde_json::{json, Value};
7223 use subc_protocol::{
7224 manifest::{
7225 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7226 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7227 },
7228 session::HealthStatus,
7229 FrameType,
7230 };
7231
7232 use super::*;
7233 use crate::{
7234 forwarding::{DataRoute, DataRouteState},
7235 registry::ChannelState,
7236 router::FrameSink,
7237 stderr_tail::DEFAULT_MAX_LINE_BYTES,
7238 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7239 RouteCtx, Router,
7240 };
7241 use tokio::{
7242 sync::mpsc,
7243 time::{sleep, Instant},
7244 };
7245 use tracing::{
7246 field::{Field, Visit},
7247 Event, Subscriber,
7248 };
7249 use tracing_subscriber::{layer::Context, prelude::*, Layer};
7250
7251 fn fake_aft_stub_path() -> PathBuf {
7270 let mut path = std::env::current_exe().expect("current_exe available in tests");
7271 path.pop(); path.pop(); path.push(if cfg!(windows) {
7274 "fake-aft-stub.exe"
7275 } else {
7276 "fake-aft-stub"
7277 });
7278 assert!(
7279 path.exists(),
7280 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7281 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7282 path.display()
7283 );
7284 path
7285 }
7286
7287 fn client_retries(code: &str) -> bool {
7292 subc_protocol::error_codes::is_retryable_route_open(code)
7293 }
7294
7295 #[test]
7306 fn retryability_of_forwarding_codes_matches_the_failure() {
7307 let transient = [
7310 ForwardingError::NoModuleConnection,
7311 ForwardingError::ModuleReloading {
7312 module_id: "m".into(),
7313 },
7314 ForwardingError::StaleModuleEndpoint,
7315 ForwardingError::UnknownReservation {
7316 client_channel: 1,
7317 module_channel: 1,
7318 },
7319 ForwardingError::ConnectionClosing {
7320 connection_id: ConnectionId::new(1),
7321 },
7322 ForwardingError::ClientEgressClosed {
7323 connection_id: ConnectionId::new(1),
7324 },
7325 ForwardingError::ModuleEgressUnavailable {
7326 connection_id: ConnectionId::new(1),
7327 },
7328 ];
7329 for err in transient {
7330 let code = forwarding_error_code(&err);
7331 assert!(
7332 client_retries(code),
7333 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7334 );
7335 }
7336
7337 let permanent = [
7342 ForwardingError::ConnectionRoleConflict {
7343 connection_id: ConnectionId::new(1),
7344 },
7345 ForwardingError::ClientRouteChannelExhausted {
7346 connection_id: ConnectionId::new(1),
7347 },
7348 ForwardingError::ModuleRouteChannelExhausted {
7349 endpoint: ModuleEndpointId {
7350 connection_id: ConnectionId::new(1),
7351 generation: 1,
7352 },
7353 },
7354 ForwardingError::RelayCorrelationExhausted,
7355 ForwardingError::RouteOpenBuild("x".into()),
7356 ForwardingError::Poisoned,
7357 ];
7358 for err in permanent {
7359 let code = forwarding_error_code(&err);
7360 assert!(
7361 !client_retries(code),
7362 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7363 );
7364 }
7365 }
7366
7367 #[tokio::test]
7395 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7396 let handler = ControlHandler::default();
7397 let frame =
7398 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7399
7400 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7404 assert_eq!(
7405 stamped,
7406 Principal::Direct,
7407 "a caller that proved nothing must not be stamped as a supervised module"
7408 );
7409
7410 let forged = handler
7415 .route_open_principal(
7416 &frame,
7417 Some(ConsumerIdentity {
7418 module_id: "aft".to_string(),
7419 launch_nonce: "not-a-real-nonce".to_string(),
7420 }),
7421 )
7422 .unwrap();
7423 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7424 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7425 }
7426
7427 #[test]
7444 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7445 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7446 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7447 let ClientControlRequest::RouteOpen {
7448 consumer_identity, ..
7449 } = parsed
7450 else {
7451 panic!("route.open body must parse as RouteOpen");
7452 };
7453 assert_eq!(
7454 consumer_identity,
7455 Some(ConsumerIdentity {
7456 module_id: "aft".to_string(),
7457 launch_nonce: "n".to_string(),
7458 }),
7459 "the wire field name must reach the value route_open_principal reads"
7460 );
7461 }
7462
7463 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7464 ModuleManifest::builder(module_id, "0.1.0")
7465 .protocol_ver(protocol_ver)
7466 .provides(vec![ProviderRole::ToolProvider {
7467 tools: vec![Tool {
7468 name: "read".to_string(),
7469 description: None,
7470 execution_mode: ExecutionMode::Pure,
7471 schema: json!({"type": "object"}),
7472 }],
7473 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7474 concurrency: Concurrency::ModuleManaged,
7475 emits_push: true,
7476 sub_supervises: true,
7477 }])
7478 .build()
7479 }
7480
7481 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7482 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7483 }
7484
7485 fn hello_frame_with_control_ops(
7486 module_id: &str,
7487 protocol_ver: u8,
7488 corr: u64,
7489 control_ops: Option<Vec<String>>,
7490 ) -> Frame {
7491 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7492 }
7493
7494 fn hello_frame_with_nonce(
7495 module_id: &str,
7496 protocol_ver: u8,
7497 corr: u64,
7498 launch_nonce: Option<&str>,
7499 ) -> Frame {
7500 hello_frame_full(
7501 module_id,
7502 protocol_ver,
7503 corr,
7504 None,
7505 launch_nonce.map(ToOwned::to_owned),
7506 )
7507 }
7508
7509 fn hello_frame_full(
7510 module_id: &str,
7511 protocol_ver: u8,
7512 corr: u64,
7513 control_ops: Option<Vec<String>>,
7514 launch_nonce: Option<String>,
7515 ) -> Frame {
7516 let body = serde_json::to_vec(&ModuleHelloBody {
7517 manifest: manifest(module_id, protocol_ver),
7518 protocol_ver,
7519 control_ops,
7520 launch_nonce,
7521 })
7522 .unwrap();
7523 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7524 }
7525
7526 fn non_routable_hello_frame_with_control_ops(
7527 module_id: &str,
7528 corr: u64,
7529 control_ops: Option<Vec<String>>,
7530 ) -> Frame {
7531 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
7532 manifest.provides.clear();
7533 let body = serde_json::to_vec(&ModuleHelloBody {
7534 manifest,
7535 protocol_ver: PROTOCOL_VERSION,
7536 control_ops,
7537 launch_nonce: None,
7538 })
7539 .unwrap();
7540 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7541 }
7542
7543 fn capability_grammar_hello_frame(
7544 capabilities: Value,
7545 runtime_computed: Option<Value>,
7546 corr: u64,
7547 ) -> Frame {
7548 let mut body = serde_json::to_value(ModuleHelloBody {
7549 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
7550 protocol_ver: PROTOCOL_VERSION,
7551 control_ops: None,
7552 launch_nonce: None,
7553 })
7554 .expect("HELLO body serializes");
7555 body["manifest"]["capabilities"] = capabilities;
7556 if let Some(runtime_computed) = runtime_computed {
7557 body["runtime_computed"] = runtime_computed;
7558 }
7559 Frame::build(
7560 FrameType::Hello,
7561 control_flags(),
7562 0,
7563 0,
7564 corr,
7565 serde_json::to_vec(&body).expect("HELLO body reserializes"),
7566 )
7567 .expect("HELLO frame builds")
7568 }
7569
7570 fn channel_request(channel: u16, corr: u64) -> Frame {
7571 Frame::build(
7572 FrameType::Request,
7573 Flags::new(true, Priority::Interactive, false),
7574 channel,
7575 0,
7576 corr,
7577 b"opaque".to_vec(),
7578 )
7579 .unwrap()
7580 }
7581
7582 fn route_ctx(
7583 connection_id: ConnectionId,
7584 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7585 let (tx, rx) = mpsc::channel(8);
7586 (
7587 RouteCtx {
7588 connection_id,
7589 egress: FrameSink::new(tx),
7590 },
7591 rx,
7592 )
7593 }
7594
7595 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7596 serde_json::from_slice(&frame.body).unwrap()
7597 }
7598
7599 async fn hello_via_sink(
7604 handler: &ControlHandler,
7605 ctx: &RouteCtx,
7606 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7607 hello: Frame,
7608 ) -> Frame {
7609 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7610 assert!(
7611 replies.is_empty(),
7612 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7613 );
7614 let ack = rx
7615 .try_recv()
7616 .expect("HELLO_ACK is queued on the module sink")
7617 .frame;
7618 assert_eq!(ack.header.ty, FrameType::HelloAck);
7619 ack
7620 }
7621
7622 fn parse_error(frame: &Frame) -> Value {
7623 serde_json::from_slice(&frame.body).unwrap()
7624 }
7625
7626 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7627 serde_json::from_slice(&frame.body).unwrap()
7628 }
7629
7630 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7631 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7632 route_channel,
7633 route_epoch: 0,
7634 kind,
7635 })
7636 .unwrap();
7637 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7638 }
7639
7640 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7641 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7642 module_id: module_id.to_string(),
7643 })
7644 .unwrap();
7645 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7646 }
7647
7648 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7649 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7650 }
7651
7652 fn route_open_frame_with_consumer_capabilities(
7653 corr: u64,
7654 module_id: &str,
7655 project_root: ScratchDir,
7656 consumer_capabilities: Option<Vec<String>>,
7657 ) -> Frame {
7658 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7659 target: RouteTarget::ToolProvider {
7660 module_id: module_id.to_string(),
7661 },
7662 identity: BindIdentity::new(
7663 project_root.path().to_path_buf(),
7664 "unit".to_string(),
7665 "session".to_string(),
7666 ),
7667 consumer_identity: None,
7668 consumer_capabilities,
7669 role_versions: None,
7670 admission_facts: None,
7671 scope: None,
7672 })
7673 .unwrap();
7674 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7675 }
7676
7677 fn route_open_frame_with_role_versions(
7678 corr: u64,
7679 module_id: &str,
7680 project_root: ScratchDir,
7681 role_versions: Option<BTreeMap<String, String>>,
7682 ) -> Frame {
7683 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7684 target: RouteTarget::ToolProvider {
7685 module_id: module_id.to_string(),
7686 },
7687 identity: BindIdentity::new(
7688 project_root.path().to_path_buf(),
7689 "unit".to_string(),
7690 format!("session-{corr}"),
7691 ),
7692 consumer_identity: None,
7693 consumer_capabilities: None,
7694 role_versions,
7695 admission_facts: None,
7696 scope: None,
7697 })
7698 .unwrap();
7699 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7700 }
7701
7702 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
7703 entries
7704 .iter()
7705 .map(|(role, version)| (role.to_string(), version.to_string()))
7706 .collect()
7707 }
7708
7709 fn route_open_frame_with_admission_facts(
7710 corr: u64,
7711 module_id: &str,
7712 project_root: ScratchDir,
7713 consumer_identity: Option<subc_control::ConsumerIdentity>,
7714 facts: Option<Value>,
7715 ) -> Frame {
7716 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
7717 target: RouteTarget::ToolProvider {
7718 module_id: module_id.to_string(),
7719 },
7720 identity: BindIdentity::new(
7721 project_root.path().to_path_buf(),
7722 "unit".to_string(),
7723 format!("session-{corr}"),
7724 ),
7725 consumer_identity,
7726 consumer_capabilities: None,
7727 role_versions: None,
7728 admission_facts: facts,
7729 scope: None,
7730 })
7731 .unwrap();
7732 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7733 }
7734
7735 #[derive(Clone, Default)]
7736 struct EventCapture {
7737 events: Arc<Mutex<Vec<CapturedEvent>>>,
7738 }
7739
7740 #[derive(Clone, Debug)]
7741 struct CapturedEvent {
7742 target: String,
7743 level: tracing::Level,
7744 fields: BTreeMap<String, String>,
7745 }
7746
7747 impl EventCapture {
7748 fn events(&self) -> Vec<CapturedEvent> {
7749 self.events.lock().unwrap().clone()
7750 }
7751 }
7752
7753 impl<S> Layer<S> for EventCapture
7754 where
7755 S: Subscriber,
7756 {
7757 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
7758 let mut visitor = EventFieldVisitor::default();
7759 event.record(&mut visitor);
7760 self.events.lock().unwrap().push(CapturedEvent {
7761 target: event.metadata().target().to_string(),
7762 level: *event.metadata().level(),
7763 fields: visitor.fields,
7764 });
7765 }
7766 }
7767
7768 #[derive(Default)]
7769 struct EventFieldVisitor {
7770 fields: BTreeMap<String, String>,
7771 }
7772
7773 impl Visit for EventFieldVisitor {
7774 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
7775 self.fields
7776 .insert(field.name().to_string(), format!("{value:?}"));
7777 }
7778 }
7779
7780 fn health_response(corr: u64, status: HealthStatus) -> Frame {
7781 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
7782 status,
7783 detail: Some("warming".to_string()),
7784 metrics: Some(json!({"queue_depth": 3})),
7785 })
7786 .unwrap();
7787 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7788 }
7789
7790 fn route_bind_ack(corr: u64) -> Frame {
7791 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
7792 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
7793 }
7794
7795 fn unique_project_root(label: &str) -> ScratchDir {
7796 ScratchDir::new(label)
7797 }
7798
7799 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
7800 match parse_route_poll(frame) {
7801 ClientControlResponse::RoutePoll {
7802 status: None,
7803 live: Some(live),
7804 ..
7805 } => assert_eq!(live, expected_live),
7806 other => panic!("unexpected route.poll response: {other:?}"),
7807 }
7808 }
7809
7810 fn bind_liveness_route(
7811 registry: &Registry,
7812 forwarding: &ForwardingTable,
7813 module_id: &str,
7814 ) -> (RouteCtx, u16, u32) {
7815 let module_connection = ConnectionId::new(101);
7816 let client_connection = ConnectionId::new(202);
7817 let registration = registry
7818 .register_with_control_ops(
7819 manifest(module_id, PROTOCOL_VERSION),
7820 PROTOCOL_VERSION,
7821 module_connection,
7822 module_baseline_control_ops(),
7823 )
7824 .unwrap();
7825 let (module_tx, _module_rx) = mpsc::channel(8);
7826 let endpoint = forwarding
7827 .register_module_connection(
7828 module_connection,
7829 module_id.to_string(),
7830 PROTOCOL_VERSION,
7831 manifest_concurrency(®istration.manifest),
7832 FrameSink::new(module_tx),
7833 )
7834 .unwrap();
7835 let (client_ctx, _client_rx) = route_ctx(client_connection);
7836 let pending = forwarding
7837 .begin_route_bind_relay_for_test(
7838 client_connection,
7839 client_ctx.egress.clone(),
7840 1,
7841 module_id,
7842 )
7843 .unwrap();
7844 assert_eq!(pending.endpoint, endpoint);
7845 let route_channel = pending.client_channel;
7846 let route_epoch = pending.client_epoch;
7847 forwarding
7848 .complete_pending_relay(
7849 module_connection,
7850 pending.corr,
7851 RouteBindRelayOutcome::Accepted,
7852 )
7853 .unwrap();
7854 (client_ctx, route_channel, route_epoch)
7855 }
7856
7857 struct FakeProcessLiveness {
7858 live: Option<bool>,
7859 }
7860
7861 impl ModuleProcessLiveness for FakeProcessLiveness {
7862 fn process_live(&self, _module_id: &str) -> Option<bool> {
7863 self.live
7864 }
7865 }
7866
7867 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
7868 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
7869 {
7870 let registry = Arc::new(Registry::default());
7871 let supervisor_handle = SupervisorHandle::new();
7872 let supervisor = Supervisor::new_for_test(
7873 Arc::clone(®istry),
7874 RestartPolicy::new(1, Duration::from_millis(10)),
7875 )
7876 .with_handle(supervisor_handle.clone());
7877 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
7878 let module = supervisor
7879 .spawn(ModuleSpec {
7880 module_id: "stderr-tail-wire".to_string(),
7881 program: fake_aft_stub_path(),
7882 args: Vec::new(),
7883 env: vec![
7884 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
7885 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
7886 ],
7887 reserved: false,
7888 reserved_prefixes: Vec::new(),
7889 protocol: ModuleProtocol::Subc,
7890 overlap: Default::default(),
7891 })
7892 .unwrap();
7893
7894 let deadline = Instant::now() + Duration::from_secs(5);
7895 loop {
7896 let tail = module.stderr_tail(None, None);
7897 if tail
7898 .entries
7899 .iter()
7900 .any(|entry| matches!(entry, TailEntry::ProcessStart))
7901 && tail.entries.iter().any(|entry| {
7902 matches!(
7903 entry,
7904 TailEntry::Line {
7905 truncated: true,
7906 ..
7907 }
7908 )
7909 })
7910 {
7911 break;
7912 }
7913 assert!(
7914 Instant::now() < deadline,
7915 "module did not produce a truncated line and restart boundary: {tail:?}"
7916 );
7917 sleep(Duration::from_millis(10)).await;
7918 }
7919
7920 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
7921 let request = ClientControlRequest::SupervisorStderrTail {
7922 module_id: "stderr-tail-wire".to_string(),
7923 max_lines: None,
7924 max_bytes: None,
7925 };
7926 let frame = Frame::build(
7927 FrameType::Request,
7928 control_flags(),
7929 0,
7930 0,
7931 1,
7932 serde_json::to_vec(&request).unwrap(),
7933 )
7934 .unwrap();
7935 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
7936 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
7937 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
7938 serde_json::from_slice(&responses[0].body).unwrap()
7939 else {
7940 panic!("expected supervisor.stderr_tail response");
7941 };
7942
7943 assert!(
7944 tail.entries
7945 .iter()
7946 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
7947 "the control response lost the restart boundary"
7948 );
7949 let Some(StderrTailEntry::Line {
7950 text,
7951 truncated,
7952 at_ms,
7953 }) = tail.entries.iter().find(|entry| {
7954 matches!(
7955 entry,
7956 StderrTailEntry::Line {
7957 truncated: true,
7958 ..
7959 }
7960 )
7961 })
7962 else {
7963 panic!("the control response lost the truncated line");
7964 };
7965 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
7966 assert!(*truncated);
7967 assert!(
7968 at_ms.is_some(),
7969 "the control response lost the line's capture time"
7970 );
7971 }
7972
7973 #[tokio::test(flavor = "current_thread")]
7978 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
7979 let dir = ScratchDir::new("terminals-off-worker");
7980 let journal_path = dir.join("terminals.jsonl");
7981 let registry = Arc::new(Registry::default());
7982 let supervisor_handle = SupervisorHandle::new();
7983 let supervisor =
7984 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
7985 .with_handle(supervisor_handle.clone())
7986 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
7987 let module = supervisor
7988 .spawn(ModuleSpec {
7989 module_id: "terminal-off-worker".to_string(),
7990 program: fake_aft_stub_path(),
7991 args: Vec::new(),
7992 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
7993 reserved: false,
7994 reserved_prefixes: Vec::new(),
7995 protocol: ModuleProtocol::Subc,
7996 overlap: Default::default(),
7997 })
7998 .unwrap();
7999 let deadline = Instant::now() + Duration::from_secs(5);
8000 while module.terminal_history().entries.len() != 2 {
8001 assert!(Instant::now() < deadline, "module did not record two exits");
8002 sleep(Duration::from_millis(10)).await;
8003 }
8004
8005 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8006 let handler =
8007 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8008 let frame = Frame::build(
8009 FrameType::Request,
8010 control_flags(),
8011 0,
8012 0,
8013 1,
8014 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8015 module_id: "terminal-off-worker".to_string(),
8016 })
8017 .unwrap(),
8018 )
8019 .unwrap();
8020 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8021 let spawned_at = std::time::Instant::now();
8022 let read = tokio::spawn({
8023 let handler = Arc::clone(&handler);
8024 async move { handler.handle_control_frame(&ctx, frame).await }
8025 });
8026 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8029 .await
8030 .unwrap()
8031 .expect("the history read reached its pause");
8032 let elapsed = spawned_at.elapsed();
8033 assert!(
8034 elapsed < Duration::from_secs(2) && !read.is_finished(),
8035 "this task could not run while the history read was paused \
8036 (resumed after {elapsed:?}, read finished: {})",
8037 read.is_finished()
8038 );
8039
8040 drop(release);
8041 let responses = read.await.unwrap().unwrap();
8042 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8043 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8044 panic!("expected supervisor.terminals response");
8045 };
8046 assert_eq!(terminals.entries.len(), 2);
8047 assert_eq!(terminals.journal_skipped_lines, 0);
8048 assert_eq!(terminals.journal_read_errors, 0);
8049 }
8050
8051 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8052 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8053 let registry = Arc::new(Registry::default());
8054 let supervisor_handle = SupervisorHandle::new();
8055 let supervisor =
8056 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8057 .with_handle(supervisor_handle.clone());
8058 let module = supervisor
8059 .spawn(ModuleSpec {
8060 module_id: "terminal-golden".to_string(),
8061 program: fake_aft_stub_path(),
8062 args: Vec::new(),
8063 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8064 reserved: false,
8065 reserved_prefixes: Vec::new(),
8066 protocol: ModuleProtocol::Subc,
8067 overlap: Default::default(),
8068 })
8069 .unwrap();
8070
8071 let deadline = Instant::now() + Duration::from_secs(5);
8072 while module.terminal_history().entries.len() != 2 {
8073 assert!(
8074 Instant::now() < deadline,
8075 "module did not retain two terminal exits: {:?}",
8076 module.terminal_history()
8077 );
8078 sleep(Duration::from_millis(10)).await;
8079 }
8080
8081 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8082 let request = ClientControlRequest::SupervisorTerminals {
8083 module_id: "terminal-golden".to_string(),
8084 };
8085 let frame = Frame::build(
8086 FrameType::Request,
8087 control_flags(),
8088 0,
8089 0,
8090 1,
8091 serde_json::to_vec(&request).unwrap(),
8092 )
8093 .unwrap();
8094 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8095 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8096 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8097 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8098 panic!("expected supervisor.terminals response");
8099 };
8100 assert_eq!(terminals.entries.len(), 2);
8101 assert_eq!(terminals.dropped, 0);
8102
8103 let mut rendered = serde_json::to_value(response).unwrap();
8104 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8107 for (index, entry) in rendered["entries"]
8108 .as_array_mut()
8109 .expect("terminal response entries array")
8110 .iter_mut()
8111 .enumerate()
8112 {
8113 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8114 }
8115
8116 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8117 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8118 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8119 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8120 std::fs::write(&golden_path, &serialized).unwrap();
8121 }
8122 let expected: Value =
8123 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8124 assert_eq!(rendered, expected);
8125 }
8126
8127 #[test]
8128 fn hello_registers_manifest_and_returns_ack() {
8129 let registry = Arc::new(Registry::default());
8130 let handler = ControlHandler::new(Arc::clone(®istry));
8131 let conn = ConnectionId::new(1);
8132
8133 let responses = handler
8134 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8135 .unwrap();
8136
8137 assert_eq!(responses.len(), 1);
8138 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8139 assert_eq!(responses[0].header.channel, 0);
8140 assert_eq!(responses[0].header.corr, 7);
8141 let ack = parse_ack(&responses[0]);
8142 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8143 assert!(ack
8144 .subc_capabilities
8145 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8146 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8147 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8148 assert!(ack
8149 .subc_ops
8150 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8151 assert!(ack
8152 .subc_ops
8153 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8154
8155 let registration = registry.get_module("aft").unwrap().unwrap();
8156 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8157 assert_eq!(registration.state, ChannelState::Active);
8158 assert_eq!(registration.connection_id, conn);
8159 assert_eq!(registration.control_ops, module_baseline_control_ops());
8160 }
8161
8162 #[test]
8163 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8164 let invalid_identifiers = [
8165 ("case_change", "credentials-Provider/v1"),
8166 ("leading_zero", "credentials-provider/v01"),
8167 ("trailing_hyphen", "credentials-provider-/v1"),
8168 ("consecutive_hyphens", "credentials--provider/v1"),
8169 ("uppercase", "Credentials-provider/v1"),
8170 ("missing_v", "credentials-provider/1"),
8171 ("whitespace", "credentials provider/v1"),
8172 ("zero_version", "credentials-provider/v0"),
8173 ("out_of_range_version", "credentials-provider/v4294967296"),
8174 (
8175 "overlength_name",
8176 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8177 ),
8178 ];
8179 let mut cases = invalid_identifiers
8180 .into_iter()
8181 .map(|(name, identifier)| {
8182 (
8183 format!("identifier_{name}"),
8184 "capabilities.provides[0]".to_string(),
8185 identifier.to_string(),
8186 json!({ "provides": [identifier] }),
8187 None,
8188 )
8189 })
8190 .collect::<Vec<_>>();
8191 cases.extend([
8192 (
8193 "unknown_need".to_string(),
8194 "capabilities.requires[0].need".to_string(),
8195 "deferred".to_string(),
8196 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8197 None,
8198 ),
8199 (
8200 "duplicate_provides".to_string(),
8201 "capabilities.provides[1]".to_string(),
8202 "credentials-provider/v1".to_string(),
8203 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8204 None,
8205 ),
8206 (
8207 "duplicate_must_never_reach".to_string(),
8208 "capabilities.must_never_reach[1]".to_string(),
8209 "credentials-provider/v1".to_string(),
8210 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8211 None,
8212 ),
8213 (
8214 "duplicate_requires_same_need".to_string(),
8215 "capabilities.requires[1]".to_string(),
8216 "credentials-provider/v1".to_string(),
8217 json!({ "requires": [
8218 { "capability": "credentials-provider/v1", "need": "required" },
8219 { "capability": "credentials-provider/v1", "need": "required" }
8220 ] }),
8221 None,
8222 ),
8223 (
8224 "duplicate_requires_conflicting_need".to_string(),
8225 "capabilities.requires[1]".to_string(),
8226 "credentials-provider/v1".to_string(),
8227 json!({ "requires": [
8228 { "capability": "credentials-provider/v1", "need": "required" },
8229 { "capability": "credentials-provider/v1", "need": "optional" }
8230 ] }),
8231 None,
8232 ),
8233 (
8234 "capabilities_root_pointer".to_string(),
8235 "runtime_computed[0]".to_string(),
8236 "/capabilities".to_string(),
8237 json!({}),
8238 Some(json!(["/capabilities"])),
8239 ),
8240 (
8241 "capabilities_descendant_pointer".to_string(),
8242 "runtime_computed[0]".to_string(),
8243 "/capabilities/provides".to_string(),
8244 json!({}),
8245 Some(json!(["/capabilities/provides"])),
8246 ),
8247 (
8248 "malformed_pointer_without_leading_slash".to_string(),
8249 "runtime_computed[0]".to_string(),
8250 "capabilities".to_string(),
8251 json!({}),
8252 Some(json!(["capabilities"])),
8253 ),
8254 (
8255 "malformed_pointer_escape".to_string(),
8256 "runtime_computed[0]".to_string(),
8257 "/roles/~2/tools".to_string(),
8258 json!({}),
8259 Some(json!(["/roles/~2/tools"])),
8260 ),
8261 (
8262 "unknown_capabilities_field".to_string(),
8263 "capabilities.future".to_string(),
8264 "<array>".to_string(),
8265 json!({ "future": [] }),
8266 None,
8267 ),
8268 ]);
8269
8270 for (index, (name, field, value, capabilities, runtime_computed)) in
8271 cases.into_iter().enumerate()
8272 {
8273 let registry = Arc::new(Registry::default());
8274 let handler = ControlHandler::new(Arc::clone(®istry));
8275 let response = handler
8276 .handle_control(
8277 ConnectionId::new((index + 1) as u64),
8278 capability_grammar_hello_frame(
8279 capabilities,
8280 runtime_computed,
8281 index as u64 + 1,
8282 ),
8283 )
8284 .expect("invalid HELLO returns a refusal");
8285
8286 assert_eq!(response.len(), 1, "{name} must emit one refusal");
8287 let error = parse_error(&response[0]);
8288 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8289 let message = error["message"]
8290 .as_str()
8291 .expect("error message is a string");
8292 assert!(
8293 message.contains(&field),
8294 "{name}: field missing from {message}"
8295 );
8296 assert!(
8297 message.contains(&value),
8298 "{name}: value missing from {message}"
8299 );
8300 assert_eq!(
8301 registry
8302 .active_registration_count()
8303 .expect("registry reads"),
8304 0,
8305 "{name}: refused HELLO must not create a catalog entry"
8306 );
8307 }
8308 }
8309
8310 #[test]
8311 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8312 let registry = Arc::new(Registry::default());
8313 let handler = ControlHandler::new(Arc::clone(®istry));
8314 let capabilities = json!({
8315 "provides": ["credentials-provider/v1"],
8316 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8317 "must_never_reach": ["federation-transport/v1"]
8318 });
8319 let response = handler
8320 .handle_control(
8321 ConnectionId::new(99),
8322 capability_grammar_hello_frame(
8323 capabilities.clone(),
8324 Some(json!(["/roles/0/tools"])),
8325 99,
8326 ),
8327 )
8328 .expect("valid HELLO registers");
8329 assert_eq!(response[0].header.ty, FrameType::HelloAck);
8330
8331 let request = Frame::build(
8332 FrameType::Request,
8333 control_flags(),
8334 0,
8335 0,
8336 100,
8337 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8338 .expect("catalog request serializes"),
8339 )
8340 .expect("catalog request frame builds");
8341 let response = handler
8342 .handle_catalog_list(request, None)
8343 .expect("catalog list succeeds");
8344 let ClientControlResponse::CatalogList { modules, .. } =
8345 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8346 else {
8347 panic!("catalog request must return catalog.list");
8348 };
8349 assert_eq!(modules.len(), 1);
8350 assert_eq!(
8351 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8352 capabilities
8353 );
8354 }
8355
8356 #[test]
8357 fn catalog_list_mirrors_management_operation_description() {
8358 let registry = Arc::new(Registry::default());
8359 let handler = ControlHandler::new(Arc::clone(®istry));
8360 let description = "List managed records and return their identifiers and metadata.";
8361 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8362 manifest.provides = vec![ProviderRole::ManagementSurface {
8363 operations: vec![ManagementOperation {
8364 name: "records.list".to_string(),
8365 kind: ManagementOperationKind::Query,
8366 description: Some(description.to_string()),
8367 }],
8368 config_schema: json!({"type": "object"}),
8369 observability: vec![ObservabilitySurface {
8370 name: "records.stats".to_string(),
8371 kind: ObservabilityKind::Snapshot,
8372 }],
8373 identity_scope: vec![IdentityScope::Project],
8374 concurrency: Concurrency::ModuleManaged,
8375 }];
8376 registry
8377 .register_with_control_ops(
8378 manifest,
8379 PROTOCOL_VERSION,
8380 ConnectionId::new(99),
8381 Vec::new(),
8382 )
8383 .expect("described management manifest registers");
8384
8385 let request = Frame::build(
8386 FrameType::Request,
8387 control_flags(),
8388 0,
8389 0,
8390 100,
8391 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8392 .expect("catalog request serializes"),
8393 )
8394 .expect("catalog request frame builds");
8395 let response = handler
8396 .handle_catalog_list(request, None)
8397 .expect("catalog list succeeds");
8398 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8399 assert_eq!(
8400 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8401 "catalog.list must preserve the declared operation description verbatim"
8402 );
8403 }
8404
8405 #[test]
8406 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8407 let registry = Arc::new(Registry::default());
8408 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8409 [("vault".to_string(), true), ("squatter".to_string(), true)],
8410 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8411 );
8412 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8413 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8414 provides: vec!["credentials-provider/v1".to_string()],
8415 requires: Vec::new(),
8416 must_never_reach: Vec::new(),
8417 });
8418 let frame = Frame::build(
8419 FrameType::Hello,
8420 control_flags(),
8421 0,
8422 0,
8423 77,
8424 serde_json::to_vec(&ModuleHelloBody {
8425 manifest: squatter,
8426 protocol_ver: PROTOCOL_VERSION,
8427 control_ops: None,
8428 launch_nonce: None,
8429 })
8430 .expect("HELLO serializes"),
8431 )
8432 .expect("HELLO frame builds");
8433 let response = handler
8434 .handle_control(ConnectionId::new(77), frame)
8435 .expect("reserved claim receives a typed refusal");
8436 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8437 assert_eq!(
8438 registry
8439 .active_registration_count()
8440 .expect("registry reads"),
8441 0,
8442 "a reserved capability refusal must not leave a catalog entry"
8443 );
8444 }
8445
8446 #[test]
8447 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8448 for settlement in ["timeout", "inconclusive", "drop"] {
8449 let breakers = RouteBindBreakers::default();
8450 let RouteBindAdmission::Admitted {
8451 guard: mut old,
8452 probe: false,
8453 } = breakers.admit("prov")
8454 else {
8455 panic!("ordinary relay admitted")
8456 };
8457 let RouteBindAdmission::Admitted {
8458 guard: mut opener, ..
8459 } = breakers.admit("prov")
8460 else {
8461 panic!("second relay admitted")
8462 };
8463 assert!(
8464 !opener
8465 .record_timeout(1, Duration::ZERO)
8466 .unwrap()
8467 .reopened_after_probe
8468 );
8469 let RouteBindAdmission::Admitted {
8470 guard: mut probe,
8471 probe: true,
8472 } = breakers.admit("prov")
8473 else {
8474 panic!("one cooldown probe admitted")
8475 };
8476 match settlement {
8477 "timeout" => assert!(
8478 !old.record_timeout(1, Duration::ZERO)
8479 .unwrap()
8480 .reopened_after_probe
8481 ),
8482 "inconclusive" => old.record_inconclusive(),
8483 "drop" => drop(old),
8484 _ => unreachable!(),
8485 }
8486 assert!(
8487 matches!(
8488 breakers.admit("prov"),
8489 RouteBindAdmission::Refused {
8490 probe_in_flight: true,
8491 ..
8492 }
8493 ),
8494 "{settlement} of a pre-open relay cannot release the real probe"
8495 );
8496 assert!(
8497 probe
8498 .record_timeout(1, Duration::ZERO)
8499 .unwrap()
8500 .reopened_after_probe
8501 );
8502 assert!(matches!(
8503 breakers.admit("prov"),
8504 RouteBindAdmission::Admitted { probe: true, .. }
8505 ));
8506 }
8507 let breakers = RouteBindBreakers::default();
8508 let admit = || match breakers.admit("prov") {
8509 RouteBindAdmission::Admitted { guard, .. } => guard,
8510 _ => panic!("relay admitted"),
8511 };
8512 admit().record_timeout(1, Duration::ZERO);
8513 let mut old_probe = admit();
8514 breakers.reset_for_new_module_connection("prov");
8515 admit().record_timeout(1, Duration::ZERO);
8516 let _new_probe = admit();
8517 old_probe.record_inconclusive();
8518 assert!(matches!(
8519 breakers.admit("prov"),
8520 RouteBindAdmission::Refused {
8521 probe_in_flight: true,
8522 ..
8523 }
8524 ));
8525 }
8526
8527 #[tokio::test]
8528 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
8529 for candidate in [false, true] {
8530 let registry = Arc::new(Registry::default());
8531 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8532 [("vault".to_string(), true), ("squatter".to_string(), true)],
8533 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8534 );
8535 let conn = ConnectionId::new(77);
8536 let (ctx, mut rx) = route_ctx(conn);
8537 let initial = capability_manifest("squatter", &[], &[]);
8538 if candidate {
8539 registry
8540 .register_candidate_with_control_ops(
8541 initial.clone(),
8542 PROTOCOL_VERSION,
8543 conn,
8544 module_baseline_control_ops(),
8545 )
8546 .unwrap();
8547 handler
8548 .forwarding
8549 .register_candidate_module_connection(
8550 conn,
8551 "squatter".to_string(),
8552 PROTOCOL_VERSION,
8553 manifest_concurrency(&initial),
8554 ctx.egress.clone(),
8555 )
8556 .unwrap();
8557 } else {
8558 hello_via_sink(
8559 &handler,
8560 &ctx,
8561 &mut rx,
8562 hello_frame_with_manifest(initial.clone(), 1),
8563 )
8564 .await;
8565 }
8566 let response = handler
8567 .handle_control_frame(
8568 &ctx,
8569 catalog_update_with_capabilities_frame(
8570 2,
8571 capability_manifest("squatter", &["credentials-provider/v1"], &[])
8572 .capabilities
8573 .unwrap(),
8574 ),
8575 )
8576 .await
8577 .unwrap();
8578 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8579 assert_eq!(
8580 registry
8581 .get_module_by_connection(conn)
8582 .unwrap()
8583 .unwrap()
8584 .manifest,
8585 initial
8586 );
8587 }
8588 }
8589
8590 #[test]
8591 fn server_describe_surfaces_required_capability_verdict_fields() {
8592 let registry = Arc::new(Registry::default());
8593 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8594 [
8595 ("consumer".to_string(), true),
8596 ("provider".to_string(), false),
8597 ],
8598 BTreeMap::new(),
8599 );
8600 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
8601 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8602 provides: Vec::new(),
8603 requires: vec![subc_protocol::manifest::CapabilityRequirement {
8604 capability: "credentials-provider/v1".to_string(),
8605 need: subc_protocol::manifest::CapabilityNeed::Required,
8606 }],
8607 must_never_reach: Vec::new(),
8608 });
8609 let hello = Frame::build(
8610 FrameType::Hello,
8611 control_flags(),
8612 0,
8613 0,
8614 78,
8615 serde_json::to_vec(&ModuleHelloBody {
8616 manifest: consumer,
8617 protocol_ver: PROTOCOL_VERSION,
8618 control_ops: None,
8619 launch_nonce: None,
8620 })
8621 .expect("HELLO serializes"),
8622 )
8623 .expect("HELLO frame builds");
8624 handler
8625 .handle_control(ConnectionId::new(78), hello)
8626 .expect("consumer registers");
8627 let describe = Frame::build(
8628 FrameType::Request,
8629 control_flags(),
8630 0,
8631 0,
8632 79,
8633 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8634 .expect("request serializes"),
8635 )
8636 .expect("describe frame builds");
8637 let response = handler
8638 .handle_server_describe(describe)
8639 .expect("server.describe succeeds");
8640 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8641 let requirement = &rendered["capability_requirements"][0];
8642 assert_eq!(requirement["consumer"], "consumer");
8643 assert_eq!(requirement["verdict"], "never_provided");
8644 assert_eq!(requirement["episode_seq"], 1);
8645 assert_eq!(requirement["config_satisfiable"], false);
8646 assert_eq!(requirement["runtime_available"], false);
8647 assert!(requirement["detail"]
8648 .as_str()
8649 .expect("detail string")
8650 .contains("credentials-provider/v1"));
8651 }
8652
8653 #[test]
8654 fn catalog_list_omits_capabilities_for_legacy_manifest() {
8655 let registry = Arc::new(Registry::default());
8656 let handler = ControlHandler::new(Arc::clone(®istry));
8657 let hello = handler
8658 .handle_control(
8659 ConnectionId::new(101),
8660 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
8661 )
8662 .expect("legacy HELLO registers");
8663 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
8664
8665 let request = Frame::build(
8666 FrameType::Request,
8667 control_flags(),
8668 0,
8669 0,
8670 102,
8671 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8672 .expect("catalog request serializes"),
8673 )
8674 .expect("catalog request frame builds");
8675 let response = handler
8676 .handle_catalog_list(request, None)
8677 .expect("catalog list succeeds");
8678 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8679 assert!(
8680 body["modules"][0].get("capabilities").is_none(),
8681 "legacy manifest must retain an absent capabilities field on catalog.list"
8682 );
8683 }
8684
8685 #[test]
8686 fn hello_ack_omits_storage_when_no_storage_config() {
8687 let registry = Arc::new(Registry::default());
8688 let handler = ControlHandler::new(Arc::clone(®istry));
8689 let responses = handler
8690 .handle_control(
8691 ConnectionId::new(1),
8692 hello_frame("aft", PROTOCOL_VERSION, 7),
8693 )
8694 .unwrap();
8695 let ack = parse_ack(&responses[0]);
8696 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
8697 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
8698 }
8699
8700 #[tokio::test]
8701 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
8702 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
8703 let registry = Arc::new(Registry::default());
8704 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
8705 let responses = handler
8706 .handle_control(
8707 ConnectionId::new(1),
8708 hello_frame("aft", PROTOCOL_VERSION, 7),
8709 )
8710 .unwrap();
8711 let ack = parse_ack(&responses[0]);
8712 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
8713
8714 let described = handler
8715 .handle_control_frame(
8716 &route_ctx(ConnectionId::new(2)).0,
8717 Frame::build(
8718 FrameType::Request,
8719 control_flags(),
8720 0,
8721 0,
8722 9,
8723 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
8724 )
8725 .unwrap(),
8726 )
8727 .await
8728 .unwrap();
8729 let ClientControlResponse::ServerDescribe { machine_id, .. } =
8730 serde_json::from_slice(&described[0].body).unwrap()
8731 else {
8732 panic!("server.describe answered with another shape");
8733 };
8734 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
8735 }
8736
8737 #[test]
8738 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
8739 let registry = Arc::new(Registry::default());
8742 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
8743 crate::daemon_config::StorageConfig::Sqlite {
8744 data_home: std::path::PathBuf::from("/data"),
8745 },
8746 ));
8747
8748 let responses = handler
8749 .handle_control(
8750 ConnectionId::new(1),
8751 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
8752 )
8753 .unwrap();
8754 let ack = parse_ack(&responses[0]);
8755 assert_eq!(
8756 ack.storage,
8757 Some(serde_json::json!({
8758 "module_id": "alfonso-routing",
8759 "storage_namespace": "default",
8760 "isolation": { "kind": "module" },
8761 "backend": {
8762 "backend": "sqlite",
8763 "path": "/data/cortexkit/alfonso-routing/store.db"
8764 }
8765 })),
8766 "the delivered descriptor is the module's own sqlite store path"
8767 );
8768 }
8769
8770 #[test]
8771 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
8772 let registry = Arc::new(Registry::default());
8773 let handler = ControlHandler::new(Arc::clone(®istry));
8774 let conn = ConnectionId::new(1);
8775 let responses = handler
8776 .handle_control(
8777 conn,
8778 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8779 )
8780 .unwrap();
8781 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8782 let registration = registry.get_module("aft").unwrap().unwrap();
8783 assert_eq!(registration.control_ops, module_baseline_control_ops());
8784
8785 let frame =
8786 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
8787 assert!(handler
8788 .guard_module_control_op(&frame, "aft", "route.bind")
8789 .unwrap()
8790 .is_none());
8791 let error = handler
8792 .guard_module_control_op(&frame, "aft", "test.synthetic")
8793 .unwrap()
8794 .expect("synthetic ungranted op should be rejected");
8795 assert_eq!(error.header.ty, FrameType::Error);
8796 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
8797 }
8798
8799 #[test]
8800 fn hello_control_ops_some_adds_optional_grants() {
8801 let registry = Arc::new(Registry::default());
8802 let handler = ControlHandler::new(Arc::clone(®istry));
8803 handler
8804 .handle_control(
8805 ConnectionId::new(1),
8806 hello_frame_with_control_ops(
8807 "aft",
8808 PROTOCOL_VERSION,
8809 7,
8810 Some(vec![
8811 "future.synthetic".to_string(),
8812 "route.bind".to_string(),
8813 ]),
8814 ),
8815 )
8816 .unwrap();
8817 let registration = registry.get_module("aft").unwrap().unwrap();
8818 assert_eq!(
8819 registration.control_ops,
8820 vec![
8821 "route.bind".to_string(),
8822 "route.status".to_string(),
8823 "future.synthetic".to_string(),
8824 ]
8825 );
8826 let frame =
8827 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
8828 assert!(handler
8829 .guard_module_control_op(&frame, "aft", "future.synthetic")
8830 .unwrap()
8831 .is_none());
8832 }
8833
8834 #[tokio::test]
8835 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
8836 let registry = Arc::new(Registry::default());
8837 let forwarding = Arc::new(ForwardingTable::default());
8838 let handler =
8839 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8840 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
8841 hello_via_sink(
8842 &handler,
8843 &module_ctx,
8844 &mut module_rx,
8845 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
8846 )
8847 .await;
8848
8849 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
8850 let responses = handler
8851 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
8852 .await
8853 .unwrap();
8854 assert_eq!(responses.len(), 1);
8855 assert_eq!(responses[0].header.ty, FrameType::Error);
8856 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
8857 assert!(module_rx.try_recv().is_err());
8858 }
8859
8860 #[tokio::test]
8861 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
8862 let registry = Arc::new(Registry::default());
8863 let forwarding = Arc::new(ForwardingTable::default());
8864 let handler =
8865 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
8866 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
8867 hello_via_sink(
8868 &handler,
8869 &module_ctx,
8870 &mut module_rx,
8871 hello_frame_with_control_ops(
8872 "aft",
8873 PROTOCOL_VERSION,
8874 7,
8875 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
8876 ),
8877 )
8878 .await;
8879
8880 let project_root = unique_project_root("demux");
8881 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
8882 let route_handler = handler.clone();
8883 let route_task = tokio::spawn(async move {
8884 route_handler
8885 .handle_control_frame(
8886 &route_client_ctx,
8887 route_open_frame(100, "aft", project_root),
8888 )
8889 .await
8890 .unwrap()
8891 });
8892 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8893 .await
8894 .unwrap()
8895 .unwrap();
8896 assert!(matches!(
8897 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
8898 ModuleControlRequest::RouteBind { .. }
8899 ));
8900
8901 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
8902 let health_handler = handler.clone();
8903 let health_task = tokio::spawn(async move {
8904 health_handler
8905 .handle_control_frame(
8906 &health_client_ctx,
8907 supervisor_health_probe_frame(101, "aft"),
8908 )
8909 .await
8910 .unwrap()
8911 });
8912 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
8913 .await
8914 .unwrap()
8915 .unwrap();
8916 assert_eq!(
8917 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
8918 ModuleControlRequest::HealthCheck {}
8919 );
8920
8921 handler
8922 .handle_control_frame(
8923 &module_ctx,
8924 health_response(health_frame.header.corr, HealthStatus::Degraded),
8925 )
8926 .await
8927 .unwrap();
8928 let health_response = health_task.await.unwrap();
8929 assert_eq!(health_response.len(), 1);
8930 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
8931 ClientControlResponse::SupervisorHealthProbe {
8932 module_id,
8933 status,
8934 detail,
8935 metrics,
8936 } => {
8937 assert_eq!(module_id, "aft");
8938 assert_eq!(status, HealthStatus::Degraded);
8939 assert_eq!(detail.as_deref(), Some("warming"));
8940 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
8941 }
8942 other => panic!("unexpected health response: {other:?}"),
8943 }
8944
8945 handler
8946 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
8947 .await
8948 .unwrap();
8949 let route_response = route_task.await.unwrap();
8950 assert!(route_response.is_empty());
8951 let published = route_client_rx.recv().await.unwrap();
8952 assert!(matches!(
8953 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
8954 ClientControlResponse::RouteOpen { .. }
8955 ));
8956 }
8957
8958 async fn relay_route_open(
8963 handler: &ControlHandler,
8964 client_connection: ConnectionId,
8965 client_egress: &FrameSink,
8966 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8967 corr: u64,
8968 module_id: &str,
8969 project_root_label: &str,
8970 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
8971 let ctx = RouteCtx {
8972 connection_id: client_connection,
8973 egress: client_egress.clone(),
8974 };
8975 let handler = handler.clone();
8976 let project_root = unique_project_root(project_root_label);
8977 let module_id = module_id.to_string();
8978 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
8979 let task = tokio::spawn(async move {
8980 let _guard = tracing::dispatcher::set_default(&dispatch);
8981 handler
8982 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
8983 .await
8984 .unwrap()
8985 });
8986 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
8987 .await
8988 .expect("module receives the relayed route.bind")
8989 .expect("module egress is open");
8990 (task, bind.frame)
8991 }
8992
8993 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
8994 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
8995 ModuleControlRequest::RouteBind {
8996 route_channel,
8997 epoch,
8998 ..
8999 } => (route_channel, epoch),
9000 other => panic!("expected a route.bind request, got {other:?}"),
9001 }
9002 }
9003
9004 fn published_route(frame: &Frame) -> (u16, u32) {
9005 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
9006 ClientControlResponse::RouteOpen {
9007 route_channel,
9008 route_epoch,
9009 } => (route_channel, route_epoch),
9010 other => panic!("expected a route.open response, got {other:?}"),
9011 }
9012 }
9013
9014 #[tokio::test]
9034 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
9035 let registry = Arc::new(Registry::default());
9036 let forwarding = Arc::new(ForwardingTable::default());
9037 let handler =
9038 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9039
9040 let module_connection = ConnectionId::new(30);
9041 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9042 hello_via_sink(
9043 &handler,
9044 &module_ctx,
9045 &mut module_rx,
9046 hello_frame("aft", PROTOCOL_VERSION, 7),
9047 )
9048 .await;
9049
9050 let dying_client = ConnectionId::new(31);
9051 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
9052
9053 let (first_task, first_bind) = relay_route_open(
9056 &handler,
9057 dying_client,
9058 &dying_ctx.egress,
9059 &mut module_rx,
9060 100,
9061 "aft",
9062 "closing-first",
9063 )
9064 .await;
9065 handler
9066 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
9067 .await
9068 .unwrap();
9069 assert!(first_task.await.unwrap().is_empty());
9070 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
9071
9072 let (second_task, second_bind) = relay_route_open(
9074 &handler,
9075 dying_client,
9076 &dying_ctx.egress,
9077 &mut module_rx,
9078 101,
9079 "aft",
9080 "closing-second",
9081 )
9082 .await;
9083 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9084
9085 assert!(forwarding
9088 .escalate_client_delivery_failure(
9089 dying_client,
9090 first_channel,
9091 first_epoch,
9092 CloseReason::new(
9093 "module_to_client_delivery_failed",
9094 "client egress refused a module frame",
9095 ),
9096 crate::forwarding::UndeliveredFrame {
9097 module_id: None,
9098 sink: &dying_ctx.egress,
9099 },
9100 )
9101 .unwrap());
9102 assert!(!dying_ctx.egress.is_closed());
9103
9104 let ack = handler
9106 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9107 .await;
9108 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9109 if module_loop_error.is_some() {
9110 handler.cleanup_connection(module_connection).unwrap();
9114 }
9115 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9119 .await
9120 .ok()
9121 .flatten();
9122
9123 assert!(
9125 registry
9126 .get_module_by_connection(module_connection)
9127 .unwrap()
9128 .is_some(),
9129 "one client's closing connection ended the shared module connection: \
9130 {module_loop_error:?}"
9131 );
9132 let cotenant = ConnectionId::new(32);
9134 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9135 let (cotenant_task, cotenant_bind) = relay_route_open(
9136 &handler,
9137 cotenant,
9138 &cotenant_ctx.egress,
9139 &mut module_rx,
9140 102,
9141 "aft",
9142 "closing-cotenant",
9143 )
9144 .await;
9145 handler
9146 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9147 .await
9148 .unwrap();
9149 assert!(cotenant_task.await.unwrap().is_empty());
9150 let (cotenant_channel, cotenant_epoch) =
9151 published_route(&cotenant_rx.recv().await.unwrap());
9152 assert!(matches!(
9153 forwarding
9154 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9155 .unwrap(),
9156 DataRoute::Client(DataRouteState::Bound(_))
9157 ));
9158
9159 let goodbye = post_ack_module_frame
9162 .expect("module receives a GOODBYE for the abandoned route channel");
9163 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9164 assert_eq!(goodbye.header.channel, abandoned_channel);
9165 assert_eq!(goodbye.header.epoch, abandoned_epoch);
9166
9167 assert!(dying_rx.try_recv().is_err());
9171 let second_response = second_task.await.unwrap();
9172 assert_eq!(second_response.len(), 1);
9173 assert_eq!(
9174 parse_error(&second_response[0])["code"],
9175 "target_unavailable"
9176 );
9177 }
9178
9179 async fn catalog_snapshot(
9182 handler: &ControlHandler,
9183 client_ctx: &RouteCtx,
9184 corr: u64,
9185 ) -> (u64, Vec<(String, bool)>) {
9186 let body =
9187 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
9188 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
9189 let reply = handler
9190 .handle_control_frame(client_ctx, request)
9191 .await
9192 .unwrap()
9193 .pop()
9194 .unwrap();
9195 let ClientControlResponse::CatalogList {
9196 generation,
9197 modules,
9198 ..
9199 } = serde_json::from_slice(&reply.body).unwrap()
9200 else {
9201 panic!("not a catalog.list reply");
9202 };
9203 (
9204 generation,
9205 modules
9206 .into_iter()
9207 .map(|entry| (entry.module_id, entry.ready))
9208 .collect(),
9209 )
9210 }
9211
9212 fn hello_held_before_forwarding() -> (
9218 ControlHandler,
9219 Arc<std::sync::Barrier>,
9220 std::thread::JoinHandle<Vec<Frame>>,
9221 RouteCtx,
9222 mpsc::Receiver<crate::router::OutboundFrame>,
9223 ) {
9224 let hold = Arc::new(std::sync::Barrier::new(2));
9225 let handler = ControlHandler::with_forwarding(
9226 Arc::new(Registry::default()),
9227 Arc::new(ForwardingTable::default()),
9228 )
9229 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
9230 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9231 let hello = {
9232 let handler = handler.clone();
9233 let module_ctx = module_ctx.clone();
9234 std::thread::spawn(move || {
9235 handler
9236 .handle_hello(
9237 module_ctx.connection_id,
9238 Some(module_ctx.egress.clone()),
9239 hello_frame("aft", PROTOCOL_VERSION, 7),
9240 )
9241 .unwrap()
9242 })
9243 };
9244 (handler, hold, hello, module_ctx, module_rx)
9245 }
9246
9247 #[tokio::test]
9256 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
9257 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
9258 let client = ConnectionId::new(41);
9259 let (client_ctx, mut client_rx) = route_ctx(client);
9260
9261 hold.wait();
9262 assert!(handler.registry.get_module("aft").unwrap().is_some());
9264 assert!(!handler
9265 .forwarding
9266 .has_live_module_connection("aft")
9267 .unwrap());
9268 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
9269 assert!(
9270 listed.is_empty(),
9271 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
9272 );
9273 let refused = handler
9274 .handle_control_frame(
9275 &client_ctx,
9276 route_open_frame(301, "aft", unique_project_root("listed-window")),
9277 )
9278 .await
9279 .unwrap();
9280 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
9281 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
9282 let duplicate = handler
9283 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
9284 .await
9285 .unwrap();
9286 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9287
9288 hold.wait();
9289 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9290 let ack = module_rx.try_recv().unwrap().frame;
9291 assert_eq!(ack.header.ty, FrameType::HelloAck);
9292 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9293 assert_eq!(listed, vec![("aft".to_string(), true)]);
9294
9295 let (open_task, bind) = relay_route_open(
9296 &handler,
9297 client,
9298 &client_ctx.egress,
9299 &mut module_rx,
9300 303,
9301 "aft",
9302 "listed-after",
9303 )
9304 .await;
9305 assert!(handler
9306 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9307 .await
9308 .unwrap()
9309 .is_empty());
9310 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
9311 published_route(&client_rx.try_recv().unwrap().frame);
9312 }
9313
9314 #[tokio::test]
9321 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
9322 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
9323 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9324 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9325
9326 hold.wait();
9327 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
9328 assert!(listed.is_empty());
9329 assert_eq!(
9330 during, before,
9331 "the generation moved while the module was not listed"
9332 );
9333
9334 hold.wait();
9335 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9336 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9337 assert_eq!(listed, vec![("aft".to_string(), true)]);
9338 assert_eq!(
9339 after,
9340 before + 1,
9341 "the module appeared without exactly one generation bump"
9342 );
9343 }
9344
9345 #[tokio::test]
9350 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
9351 let registry = Arc::new(Registry::default());
9352 let handler = ControlHandler::with_forwarding(
9353 Arc::clone(®istry),
9354 Arc::new(ForwardingTable::default()),
9355 );
9356 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9357 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9358 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9359 drop(module_rx);
9360
9361 let replies = handler
9362 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
9363 .await
9364 .unwrap();
9365 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
9366 assert!(registry.get_module("aft").unwrap().is_none());
9367 assert!(registry
9368 .get_module_by_connection(module_ctx.connection_id)
9369 .unwrap()
9370 .is_none());
9371 assert_eq!(
9372 catalog_snapshot(&handler, &client_ctx, 301).await,
9373 (before, Vec::new())
9374 );
9375 }
9376
9377 async fn route_open_held_after_bind_send() -> (
9381 ControlHandler,
9382 Arc<tokio::sync::Barrier>,
9383 RouteCtx,
9384 mpsc::Receiver<crate::router::OutboundFrame>,
9385 ) {
9386 let hold = Arc::new(tokio::sync::Barrier::new(2));
9387 let handler = ControlHandler::with_forwarding(
9388 Arc::new(Registry::default()),
9389 Arc::new(ForwardingTable::default()),
9390 )
9391 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
9392 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9393 hello_via_sink(
9394 &handler,
9395 &module_ctx,
9396 &mut module_rx,
9397 hello_frame("aft", PROTOCOL_VERSION, 7),
9398 )
9399 .await;
9400 (handler, hold, module_ctx, module_rx)
9401 }
9402
9403 #[tokio::test]
9410 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
9411 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9412 let client = ConnectionId::new(41);
9413 let (client_ctx, mut client_rx) = route_ctx(client);
9414
9415 let (open_task, bind) = relay_route_open(
9416 &handler,
9417 client,
9418 &client_ctx.egress,
9419 &mut module_rx,
9420 200,
9421 "aft",
9422 "acked-before-mark",
9423 )
9424 .await;
9425 let (module_channel, module_epoch) = route_bind_channel(&bind);
9426 hold.wait().await;
9428
9429 assert!(handler
9431 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9432 .await
9433 .unwrap()
9434 .is_empty());
9435 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
9436
9437 hold.wait().await;
9438 assert!(
9439 open_task.await.unwrap().is_empty(),
9440 "route.open answers accepted"
9441 );
9442
9443 match module_rx.try_recv() {
9446 Err(mpsc::error::TryRecvError::Empty) => {}
9447 Ok(outbound) => panic!(
9448 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
9449 outbound.frame.header.ty,
9450 outbound.frame.header.channel,
9451 outbound.frame.header.epoch
9452 ),
9453 Err(err) => panic!("module egress closed: {err}"),
9454 }
9455
9456 let router = Router::with_control_handler(Arc::new(handler.clone()));
9459 let request = Frame::build(
9460 FrameType::Request,
9461 Flags::new(true, Priority::Interactive, false),
9462 client_channel,
9463 client_epoch,
9464 201,
9465 b"first request".to_vec(),
9466 )
9467 .unwrap();
9468 router
9469 .route_for_connection(&client_ctx, request)
9470 .await
9471 .unwrap();
9472 let delivered = module_rx.try_recv().unwrap().frame;
9473 assert_eq!(delivered.header.ty, FrameType::Request);
9474 assert_eq!(
9475 (delivered.header.channel, delivered.header.epoch),
9476 (module_channel, module_epoch)
9477 );
9478 assert_eq!(delivered.body, b"first request");
9479 }
9480
9481 #[tokio::test]
9487 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
9488 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9489 let client = ConnectionId::new(42);
9490 let (client_ctx, _client_rx) = route_ctx(client);
9491
9492 let (open_task, bind) = relay_route_open(
9493 &handler,
9494 client,
9495 &client_ctx.egress,
9496 &mut module_rx,
9497 300,
9498 "aft",
9499 "released-before-mark",
9500 )
9501 .await;
9502 let (module_channel, module_epoch) = route_bind_channel(&bind);
9503 hold.wait().await;
9504
9505 handler.cleanup_connection(client).unwrap();
9506 assert!(
9507 module_rx.try_recv().is_err(),
9508 "client teardown must not GOODBYE a bind route.open is still queueing"
9509 );
9510
9511 hold.wait().await;
9512 let response = open_task.await.unwrap();
9513 assert_eq!(response.len(), 1);
9514 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9515
9516 let goodbye = module_rx
9517 .try_recv()
9518 .expect("module is told to drop the binding the daemon abandoned")
9519 .frame;
9520 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9521 assert_eq!(
9522 (goodbye.header.channel, goodbye.header.epoch),
9523 (module_channel, module_epoch)
9524 );
9525 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9526 }
9527
9528 #[tokio::test]
9533 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
9534 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9535 let client = ConnectionId::new(43);
9536 let (client_ctx, client_rx) = route_ctx(client);
9537
9538 let (open_task, bind) = relay_route_open(
9539 &handler,
9540 client,
9541 &client_ctx.egress,
9542 &mut module_rx,
9543 400,
9544 "aft",
9545 "gone-client-before-mark",
9546 )
9547 .await;
9548 let (module_channel, module_epoch) = route_bind_channel(&bind);
9549 hold.wait().await;
9550
9551 drop(client_rx);
9552 assert!(client_ctx.egress.is_closed());
9553 handler
9554 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9555 .await
9556 .unwrap();
9557
9558 hold.wait().await;
9559 let response = open_task.await.unwrap();
9560 assert_eq!(response.len(), 1);
9561 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9562
9563 let goodbye = module_rx
9564 .try_recv()
9565 .expect("module is told to drop the binding it acked")
9566 .frame;
9567 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9568 assert_eq!(
9569 (goodbye.header.channel, goodbye.header.epoch),
9570 (module_channel, module_epoch)
9571 );
9572 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9573 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
9574 }
9575
9576 #[test]
9583 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
9584 let handler = ControlHandler::default();
9585 let module_connection = ConnectionId::new(30);
9586 let client_connection = ConnectionId::new(31);
9587
9588 handler
9589 .refuse_to_end_module_connection_for_a_client(
9590 module_connection,
9591 77,
9592 ForwardingError::ConnectionClosing {
9593 connection_id: client_connection,
9594 },
9595 )
9596 .expect("a closing client must never end the module connection");
9597
9598 assert!(matches!(
9599 handler.refuse_to_end_module_connection_for_a_client(
9600 module_connection,
9601 78,
9602 ForwardingError::ConnectionClosing {
9603 connection_id: module_connection,
9604 },
9605 ),
9606 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
9607 connection_id
9608 })) if connection_id == module_connection
9609 ));
9610 assert!(matches!(
9611 handler.refuse_to_end_module_connection_for_a_client(
9612 module_connection,
9613 79,
9614 ForwardingError::Poisoned,
9615 ),
9616 Err(RouterError::Forwarding(ForwardingError::Poisoned))
9617 ));
9618 assert!(matches!(
9619 handler.refuse_to_end_module_connection_for_a_client(
9620 module_connection,
9621 80,
9622 ForwardingError::StaleModuleEndpoint,
9623 ),
9624 Err(RouterError::Forwarding(
9625 ForwardingError::StaleModuleEndpoint
9626 ))
9627 ));
9628 }
9629
9630 #[tokio::test]
9643 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
9644 let registry = Arc::new(Registry::default());
9645 let forwarding = Arc::new(ForwardingTable::default());
9646 let supervisor = SupervisorHandle::new();
9647 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9648 let handler =
9649 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9650 .with_supervisor(supervisor);
9651
9652 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9653 hello_via_sink(
9654 &handler,
9655 &target_ctx,
9656 &mut target_rx,
9657 hello_frame("target", PROTOCOL_VERSION, 1),
9658 )
9659 .await;
9660
9661 let wrong_nonce = handler
9665 .handle_control_frame(
9666 &route_ctx(ConnectionId::new(91)).0,
9667 route_open_frame_with_admission_facts(
9668 20,
9669 "target",
9670 unique_project_root("admission-facts"),
9671 Some(subc_control::ConsumerIdentity {
9672 module_id: "fed".to_string(),
9673 launch_nonce: "not-the-real-nonce".to_string(),
9674 }),
9675 None,
9676 ),
9677 )
9678 .await
9679 .unwrap();
9680 assert_eq!(
9681 parse_error(&wrong_nonce[0])["code"],
9682 "bad_consumer_identity",
9683 "a mismatched launch nonce must be refused, not stamped Reserved"
9684 );
9685
9686 let never_spawned = handler
9690 .handle_control_frame(
9691 &route_ctx(ConnectionId::new(92)).0,
9692 route_open_frame_with_admission_facts(
9693 21,
9694 "target",
9695 unique_project_root("admission-facts"),
9696 Some(subc_control::ConsumerIdentity {
9697 module_id: "never-spawned".to_string(),
9698 launch_nonce: "any-nonce".to_string(),
9699 }),
9700 None,
9701 ),
9702 )
9703 .await
9704 .unwrap();
9705 assert_eq!(
9706 parse_error(&never_spawned[0])["code"],
9707 "bad_consumer_identity",
9708 "an unspawned module_id must be refused rather than accepted for lack of a record"
9709 );
9710 }
9711
9712 #[tokio::test]
9735 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
9736 let registry = Arc::new(Registry::default());
9737 let forwarding = Arc::new(ForwardingTable::default());
9738 let supervisor = SupervisorHandle::new();
9739 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9740 let handler =
9741 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9742 .with_supervisor(supervisor);
9743
9744 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
9745 hello_via_sink(
9746 &handler,
9747 &target_ctx,
9748 &mut target_rx,
9749 hello_frame("target", PROTOCOL_VERSION, 1),
9750 )
9751 .await;
9752
9753 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
9754 let route_handler = handler.clone();
9755 let route_task = tokio::spawn(async move {
9756 route_handler
9757 .handle_control_frame(
9758 &client_ctx,
9759 route_open_frame_with_admission_facts(
9760 30,
9761 "target",
9762 unique_project_root("admission-facts"),
9763 Some(subc_control::ConsumerIdentity {
9764 module_id: "fed".to_string(),
9765 launch_nonce: "fed-nonce".to_string(),
9766 }),
9767 None,
9768 ),
9769 )
9770 .await
9771 .unwrap()
9772 });
9773
9774 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9781 .await
9782 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
9783 .expect("module control channel closed before route.bind");
9784 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
9785 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
9786 panic!("expected route.bind")
9787 };
9788 assert_eq!(
9789 principal,
9790 Some(Principal::Reserved {
9791 module_id: "fed".to_string()
9792 }),
9793 "a correctly attested consumer must be stamped Reserved for its own id"
9794 );
9795
9796 handler
9797 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
9798 .await
9799 .unwrap();
9800 assert!(route_task.await.unwrap().is_empty());
9801 assert!(
9802 matches!(
9803 serde_json::from_slice::<ClientControlResponse>(
9804 &client_rx.recv().await.unwrap().body
9805 )
9806 .unwrap(),
9807 ClientControlResponse::RouteOpen { .. }
9808 ),
9809 "the route must actually open, not merely avoid an error"
9810 );
9811 }
9812
9813 #[tokio::test(start_paused = true)]
9814 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
9815 let registry = Arc::new(Registry::default());
9816 let forwarding = Arc::new(ForwardingTable::default());
9817 let supervisor = SupervisorHandle::new();
9818 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9819 let handler =
9820 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9821 .with_supervisor(supervisor);
9822
9823 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
9824 hello_via_sink(
9825 &handler,
9826 &target_ctx,
9827 &mut target_rx,
9828 hello_frame("target", PROTOCOL_VERSION, 1),
9829 )
9830 .await;
9831
9832 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
9833 let direct_handler = handler.clone();
9834 let direct_open = tokio::spawn(async move {
9835 direct_handler
9836 .handle_control_frame(
9837 &direct_ctx,
9838 route_open_frame(2, "target", unique_project_root("route-census-direct")),
9839 )
9840 .await
9841 .unwrap()
9842 });
9843 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9844 .await
9845 .expect("no direct route.bind within 5s")
9846 .expect("target control channel closed before direct route.bind");
9847 handler
9848 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
9849 .await
9850 .unwrap();
9851 assert!(direct_open.await.unwrap().is_empty());
9852 let _ = direct_rx.recv().await.unwrap();
9853
9854 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
9855 let reserved_handler = handler.clone();
9856 let reserved_open = tokio::spawn(async move {
9857 reserved_handler
9858 .handle_control_frame(
9859 &reserved_ctx,
9860 route_open_frame_with_admission_facts(
9861 3,
9862 "target",
9863 unique_project_root("admission-facts"),
9864 Some(ConsumerIdentity {
9865 module_id: "fed".to_string(),
9866 launch_nonce: "fed-nonce".to_string(),
9867 }),
9868 None,
9869 ),
9870 )
9871 .await
9872 .unwrap()
9873 });
9874 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9875 .await
9876 .expect("no reserved route.bind within 5s")
9877 .expect("target control channel closed before reserved route.bind");
9878 handler
9879 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
9880 .await
9881 .unwrap();
9882 assert!(reserved_open.await.unwrap().is_empty());
9883 let _ = reserved_rx.recv().await.unwrap();
9884
9885 forwarding
9886 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
9887 .unwrap();
9888 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
9889 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
9890 module_id: Some("target".to_string()),
9891 })
9892 .unwrap();
9893 let census_frame =
9894 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
9895 let response = handler
9896 .handle_control_frame(&census_ctx, census_frame)
9897 .await
9898 .unwrap()
9899 .pop()
9900 .unwrap();
9901 let actual: Value = serde_json::from_slice(&response.body).unwrap();
9902 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
9903 assert!(matches!(
9904 decoded,
9905 ClientControlResponse::SupervisorRoutes { .. }
9906 ));
9907 let routes = actual["modules"][0]["routes"].as_array().unwrap();
9908 assert_eq!(routes.len(), 2);
9909 assert!(routes.iter().all(|route| route["draining"] == true));
9910 assert!(
9913 routes.iter().all(|route| route["drain_reason"] == "reload"),
9914 "draining routes must name the drain's reason: {routes:?}"
9915 );
9916 assert!(routes.iter().any(|route| {
9917 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
9918 }));
9919 assert!(routes.iter().any(|route| {
9920 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
9921 }));
9922
9923 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9924 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
9925 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9926 std::fs::write(
9927 &golden_path,
9928 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
9929 )
9930 .unwrap();
9931 }
9932 let expected: Value =
9933 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
9934 assert_eq!(actual, expected);
9935 }
9936
9937 async fn query_live_roots(
9938 handler: &ControlHandler,
9939 module_ctx: &RouteCtx,
9940 ) -> ModuleControlResponseToModule {
9941 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
9942 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
9943 let response = handler
9944 .handle_control_frame(module_ctx, frame)
9945 .await
9946 .unwrap()
9947 .pop()
9948 .unwrap();
9949 serde_json::from_slice(&response.body).unwrap()
9950 }
9951
9952 #[tokio::test(start_paused = true)]
9953 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
9954 let registry = Arc::new(Registry::default());
9955 let forwarding = Arc::new(ForwardingTable::default());
9956 let handler = ControlHandler::with_forwarding(registry, forwarding);
9957 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
9958 hello_via_sink(
9959 &handler,
9960 &target_ctx,
9961 &mut target_rx,
9962 hello_frame("target", PROTOCOL_VERSION, 1),
9963 )
9964 .await;
9965 let root = unique_project_root("live-roots-known");
9966 let path = ProjectRootId::from_path_allowing_missing(root.path())
9967 .unwrap()
9968 .as_path()
9969 .to_path_buf();
9970 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
9971 let open_handler = handler.clone();
9972 let opened = tokio::spawn(async move {
9973 open_handler
9974 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
9975 .await
9976 .unwrap()
9977 });
9978 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
9979 .await
9980 .unwrap()
9981 .unwrap();
9982 handler
9983 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
9984 .await
9985 .unwrap();
9986 assert!(opened.await.unwrap().is_empty());
9987 let _ = client_rx.recv().await.unwrap();
9988
9989 let root = unique_project_root("live-roots-pending");
9990 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
9991 .unwrap()
9992 .as_path()
9993 .to_path_buf();
9994 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
9995 let open_handler = handler.clone();
9996 let pending = tokio::spawn(async move {
9997 open_handler
9998 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
9999 .await
10000 .unwrap()
10001 });
10002 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10003 .await
10004 .unwrap()
10005 .unwrap();
10006 let actual = query_live_roots(&handler, &target_ctx).await;
10007 let ModuleControlResponseToModule::LiveRoots {
10008 roots,
10009 unknown_root_bindings,
10010 total_bindings,
10011 } = actual
10012 else {
10013 panic!("expected live roots")
10014 };
10015 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
10016 assert_eq!(unknown_root_bindings, 0);
10017 assert_eq!(
10018 roots.len(),
10019 2,
10020 "root-known arm must retain each canonical root"
10021 );
10022 assert_eq!(
10023 total_bindings,
10024 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10025 );
10026 let counts = roots
10027 .iter()
10028 .map(|root| (root.project_root.clone(), root.bound, root.pending))
10029 .collect::<Vec<_>>();
10030 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
10031 expected.sort_by(|a, b| a.0.cmp(&b.0));
10032 assert_eq!(
10033 counts, expected,
10034 "roots must sort by path and count pending separately"
10035 );
10036 handler
10037 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
10038 .await
10039 .unwrap();
10040 assert!(pending.await.unwrap().is_empty());
10041 }
10042
10043 #[tokio::test(start_paused = true)]
10044 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
10045 let forwarding = Arc::new(ForwardingTable::default());
10046 let handler =
10047 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10048 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
10049 hello_via_sink(
10050 &handler,
10051 &target_ctx,
10052 &mut target_rx,
10053 hello_frame("target", PROTOCOL_VERSION, 1),
10054 )
10055 .await;
10056 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
10057 let pending = forwarding
10058 .begin_route_bind_relay_for_test(
10059 client_ctx.connection_id,
10060 client_ctx.egress.clone(),
10061 2,
10062 "target",
10063 )
10064 .unwrap();
10065 forwarding
10066 .complete_pending_relay(
10067 target_ctx.connection_id,
10068 pending.corr,
10069 RouteBindRelayOutcome::Accepted,
10070 )
10071 .unwrap();
10072 let actual = query_live_roots(&handler, &target_ctx).await;
10073 let ModuleControlResponseToModule::LiveRoots {
10074 roots,
10075 unknown_root_bindings,
10076 total_bindings,
10077 } = actual
10078 else {
10079 panic!("expected live roots")
10080 };
10081 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
10082 assert_eq!(
10083 unknown_root_bindings, 1,
10084 "unknown-root arm must not read as no bindings"
10085 );
10086 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
10087 assert_eq!(
10088 total_bindings,
10089 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10090 );
10091 }
10092
10093 #[tokio::test(start_paused = true)]
10101 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
10102 let forwarding = Arc::new(ForwardingTable::default());
10103 let handler =
10104 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10105 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
10106 let replies = handler
10107 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
10108 .await
10109 .unwrap();
10110 let queued_by_hello = module_rx.len();
10111
10112 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
10113 let open_handler = handler.clone();
10114 let open = tokio::spawn(async move {
10115 open_handler
10116 .handle_control_frame(
10117 &client_ctx,
10118 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
10119 )
10120 .await
10121 .unwrap()
10122 });
10123 let mut spins = 0;
10125 while module_rx.len() == queued_by_hello {
10126 spins += 1;
10127 assert!(spins < 10_000, "route.open never queued a route.bind");
10128 tokio::task::yield_now().await;
10129 }
10130
10131 for reply in replies {
10133 module_ctx.egress.send(reply).await.unwrap();
10134 }
10135
10136 let first = module_rx.recv().await.unwrap().frame;
10137 assert_eq!(
10138 first.header.ty,
10139 FrameType::HelloAck,
10140 "the first frame a registering module reads must be its HELLO_ACK"
10141 );
10142 assert_eq!(first.header.corr, 7);
10143 let second = module_rx.recv().await.unwrap().frame;
10144 assert_eq!(second.header.ty, FrameType::Request);
10145 assert!(
10146 matches!(
10147 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
10148 ModuleControlRequest::RouteBind { .. }
10149 ),
10150 "the route.bind follows the ack"
10151 );
10152 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
10153
10154 handler
10155 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
10156 .await
10157 .unwrap();
10158 assert!(open.await.unwrap().is_empty());
10159 let _ = client_rx.recv().await.unwrap();
10160 }
10161
10162 #[tokio::test(start_paused = true)]
10163 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
10164 let handler = ControlHandler::with_forwarding(
10165 Arc::new(Registry::default()),
10166 Arc::new(ForwardingTable::default()),
10167 );
10168 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
10169 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
10170 hello_via_sink(
10171 &handler,
10172 &first_ctx,
10173 &mut first_rx,
10174 hello_frame("first", PROTOCOL_VERSION, 1),
10175 )
10176 .await;
10177 hello_via_sink(
10178 &handler,
10179 &second_ctx,
10180 &mut second_rx,
10181 hello_frame("second", PROTOCOL_VERSION, 2),
10182 )
10183 .await;
10184 let root = unique_project_root("second-only");
10185 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
10186 let cloned = handler.clone();
10187 let open = tokio::spawn(async move {
10188 cloned
10189 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
10190 .await
10191 .unwrap()
10192 });
10193 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
10194 .await
10195 .unwrap()
10196 .unwrap();
10197 let first = query_live_roots(&handler, &first_ctx).await;
10198 let second = query_live_roots(&handler, &second_ctx).await;
10199 assert!(
10200 matches!(
10201 first,
10202 ModuleControlResponseToModule::LiveRoots {
10203 total_bindings: 0,
10204 ..
10205 }
10206 ),
10207 "cross-module scope must not expose another module's roots"
10208 );
10209 assert!(
10210 matches!(
10211 second,
10212 ModuleControlResponseToModule::LiveRoots {
10213 total_bindings: 1,
10214 ..
10215 }
10216 ),
10217 "second module must see its pending route"
10218 );
10219 handler
10220 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
10221 .await
10222 .unwrap();
10223 assert!(open.await.unwrap().is_empty());
10224 }
10225
10226 #[tokio::test(start_paused = true)]
10227 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
10228 let handler = ControlHandler::with_forwarding(
10229 Arc::new(Registry::default()),
10230 Arc::new(ForwardingTable::default()),
10231 );
10232 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
10233 hello_via_sink(
10234 &handler,
10235 &target_ctx,
10236 &mut target_rx,
10237 hello_frame("target", PROTOCOL_VERSION, 1),
10238 )
10239 .await;
10240 let actual = query_live_roots(&handler, &target_ctx).await;
10241 let ModuleControlResponseToModule::LiveRoots {
10242 roots,
10243 unknown_root_bindings,
10244 total_bindings,
10245 } = actual
10246 else {
10247 panic!("expected live roots")
10248 };
10249 assert!(roots.is_empty());
10250 assert_eq!(unknown_root_bindings, 0);
10251 assert_eq!(total_bindings, 0);
10252 assert_eq!(
10253 total_bindings,
10254 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10255 );
10256 }
10257
10258 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
10265 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10266 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
10267 let text = std::fs::read_to_string(&path)
10268 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
10269 let vectors: Vec<(String, Value)> = text
10270 .lines()
10271 .filter(|line| !line.trim().is_empty())
10272 .map(|line| {
10273 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
10274 let id = entry["corpus_id"]
10275 .as_str()
10276 .expect("every vector carries a corpus_id")
10277 .to_string();
10278 (id, entry["package"].clone())
10279 })
10280 .collect();
10281 assert_eq!(
10285 vectors.len(),
10286 3,
10287 "vendored fed corpus changed size; re-sync from subc-federation"
10288 );
10289
10290 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
10305 let richest = vectors
10306 .iter()
10307 .filter_map(|(_, package)| package.as_object())
10308 .map(|object| {
10309 object
10310 .keys()
10311 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
10312 .count()
10313 })
10314 .max()
10315 .unwrap_or(0);
10316 assert!(
10317 richest >= 2,
10318 "vendored corpus no longer carries a package with unmodelled fields, \
10319 so the relay test can no longer distinguish a verbatim relay from a lossy one"
10320 );
10321
10322 vectors
10323 }
10324
10325 #[tokio::test]
10333 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
10334 for (corpus_id, package) in fed_admission_facts_vectors() {
10335 let registry = Arc::new(Registry::default());
10336 let forwarding = Arc::new(ForwardingTable::default());
10337 let supervisor = SupervisorHandle::new();
10338 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10339 let handler =
10340 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10341 .with_supervisor(supervisor)
10342 .with_admission_facts_config(
10343 Some("fed".to_string()),
10344 Some(vec!["target".to_string()]),
10345 );
10346
10347 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10348 hello_via_sink(
10349 &handler,
10350 &target_ctx,
10351 &mut target_rx,
10352 hello_frame("target", PROTOCOL_VERSION, 1),
10353 )
10354 .await;
10355
10356 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
10357 let route_handler = handler.clone();
10358 let expected = package.clone();
10359 let route_task = tokio::spawn(async move {
10360 route_handler
10361 .handle_control_frame(
10362 &client_ctx,
10363 route_open_frame_with_admission_facts(
10364 20,
10365 "target",
10366 unique_project_root("admission-facts"),
10367 Some(subc_control::ConsumerIdentity {
10368 module_id: "fed".to_string(),
10369 launch_nonce: "fed-nonce".to_string(),
10370 }),
10371 Some(package),
10372 ),
10373 )
10374 .await
10375 .unwrap()
10376 });
10377
10378 let bind_frame = target_rx.recv().await.unwrap();
10379 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10380 let ModuleControlRequest::RouteBind {
10381 admission_facts, ..
10382 } = bind
10383 else {
10384 panic!("{corpus_id}: expected route.bind")
10385 };
10386 assert_eq!(
10387 admission_facts,
10388 Some(expected),
10389 "{corpus_id}: relay must not add, drop or reshape any field"
10390 );
10391
10392 handler
10393 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10394 .await
10395 .unwrap();
10396 route_task.await.unwrap();
10397 }
10398 }
10399
10400 #[tokio::test]
10401 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
10402 let registry = Arc::new(Registry::default());
10403 let forwarding = Arc::new(ForwardingTable::default());
10404 let supervisor = SupervisorHandle::new();
10405 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10406 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
10407 let handler =
10408 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10409 .with_supervisor(supervisor)
10410 .with_admission_facts_config(
10411 Some("fed".to_string()),
10412 Some(vec!["target".to_string()]),
10413 );
10414
10415 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
10416 hello_via_sink(
10417 &handler,
10418 &target_ctx,
10419 &mut target_rx,
10420 hello_frame("target", PROTOCOL_VERSION, 1),
10421 )
10422 .await;
10423 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
10424 hello_via_sink(
10425 &handler,
10426 &other_ctx,
10427 &mut other_rx,
10428 hello_frame("other", PROTOCOL_VERSION, 2),
10429 )
10430 .await;
10431
10432 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
10433 let expected_facts = facts.clone();
10434 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
10435 let route_handler = handler.clone();
10436 let route_task = tokio::spawn(async move {
10437 route_handler
10438 .handle_control_frame(
10439 &client_ctx,
10440 route_open_frame_with_admission_facts(
10441 10,
10442 "target",
10443 unique_project_root("admission-facts"),
10444 Some(subc_control::ConsumerIdentity {
10445 module_id: "fed".to_string(),
10446 launch_nonce: "fed-nonce".to_string(),
10447 }),
10448 Some(facts.clone()),
10449 ),
10450 )
10451 .await
10452 .unwrap()
10453 });
10454 let bind_frame = target_rx.recv().await.unwrap();
10455 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10456 let ModuleControlRequest::RouteBind {
10457 admission_facts, ..
10458 } = bind
10459 else {
10460 panic!("expected route.bind")
10461 };
10462 assert_eq!(admission_facts, Some(expected_facts));
10463 handler
10464 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10465 .await
10466 .unwrap();
10467 assert!(route_task.await.unwrap().is_empty());
10468 assert!(matches!(
10469 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
10470 .unwrap(),
10471 ClientControlResponse::RouteOpen { .. }
10472 ));
10473
10474 let direct = handler
10475 .handle_control_frame(
10476 &route_ctx(ConnectionId::new(73)).0,
10477 route_open_frame_with_admission_facts(
10478 11,
10479 "target",
10480 unique_project_root("admission-facts"),
10481 None,
10482 Some(json!({"x": 1})),
10483 ),
10484 )
10485 .await
10486 .unwrap();
10487 assert_eq!(
10488 parse_error(&direct[0])["code"],
10489 "admission_facts_not_permitted"
10490 );
10491
10492 let different_reserved = handler
10493 .handle_control_frame(
10494 &route_ctx(ConnectionId::new(77)).0,
10495 route_open_frame_with_admission_facts(
10496 15,
10497 "target",
10498 unique_project_root("admission-facts"),
10499 Some(subc_control::ConsumerIdentity {
10500 module_id: "other".to_string(),
10501 launch_nonce: "other-nonce".to_string(),
10502 }),
10503 Some(json!({"x": 1})),
10504 ),
10505 )
10506 .await
10507 .unwrap();
10508 assert_eq!(
10509 parse_error(&different_reserved[0])["code"],
10510 "admission_facts_not_permitted"
10511 );
10512
10513 let other_target = handler
10514 .handle_control_frame(
10515 &route_ctx(ConnectionId::new(74)).0,
10516 route_open_frame_with_admission_facts(
10517 12,
10518 "other",
10519 unique_project_root("admission-facts"),
10520 Some(subc_control::ConsumerIdentity {
10521 module_id: "fed".to_string(),
10522 launch_nonce: "fed-nonce".to_string(),
10523 }),
10524 Some(json!({"x": 1})),
10525 ),
10526 )
10527 .await
10528 .unwrap();
10529 assert_eq!(
10530 parse_error(&other_target[0])["code"],
10531 "admission_facts_target_not_allowed"
10532 );
10533
10534 let nonexistent = handler
10535 .handle_control_frame(
10536 &route_ctx(ConnectionId::new(75)).0,
10537 route_open_frame_with_admission_facts(
10538 13,
10539 "missing",
10540 unique_project_root("admission-facts"),
10541 None,
10542 Some(json!({"x": 1})),
10543 ),
10544 )
10545 .await
10546 .unwrap();
10547 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
10548
10549 let described = handler
10550 .handle_control_frame(
10551 &route_ctx(ConnectionId::new(76)).0,
10552 Frame::build(
10553 FrameType::Request,
10554 control_flags(),
10555 0,
10556 0,
10557 14,
10558 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10559 )
10560 .unwrap(),
10561 )
10562 .await
10563 .unwrap();
10564 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10565 serde_json::from_slice(&described[0].body).unwrap()
10566 else {
10567 panic!("expected server.describe response")
10568 };
10569 assert!(capabilities
10570 .iter()
10571 .any(|cap| cap == "admission_facts_relay_v1"));
10572 }
10573
10574 #[tokio::test]
10575 async fn admission_facts_without_configured_carrier_are_rejected() {
10576 let registry = Arc::new(Registry::default());
10577 let forwarding = Arc::new(ForwardingTable::default());
10578 let handler = ControlHandler::with_forwarding(registry, forwarding);
10579 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
10580 hello_via_sink(
10581 &handler,
10582 &target_ctx,
10583 &mut target_rx,
10584 hello_frame("target", PROTOCOL_VERSION, 1),
10585 )
10586 .await;
10587
10588 let responses = handler
10589 .handle_control_frame(
10590 &route_ctx(ConnectionId::new(79)).0,
10591 route_open_frame_with_admission_facts(
10592 16,
10593 "target",
10594 unique_project_root("admission-facts"),
10595 None,
10596 Some(json!({"x": 1})),
10597 ),
10598 )
10599 .await
10600 .unwrap();
10601 assert_eq!(
10602 parse_error(&responses[0])["code"],
10603 "admission_facts_not_permitted"
10604 );
10605 }
10606
10607 #[tokio::test]
10608 async fn route_open_relays_consumer_capabilities_verbatim() {
10609 let registry = Arc::new(Registry::default());
10610 let forwarding = Arc::new(ForwardingTable::default());
10611 let handler =
10612 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10613 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
10614 hello_via_sink(
10615 &handler,
10616 &module_ctx,
10617 &mut module_rx,
10618 hello_frame("aft", PROTOCOL_VERSION, 7),
10619 )
10620 .await;
10621
10622 let expected = vec!["elicitation".to_string(), "roots".to_string()];
10623 let expected_for_request = expected.clone();
10624 let project_root = unique_project_root("consumer-capabilities-present");
10625 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
10626 let route_handler = handler.clone();
10627 let route_task = tokio::spawn(async move {
10628 route_handler
10629 .handle_control_frame(
10630 &client_ctx,
10631 route_open_frame_with_consumer_capabilities(
10632 401,
10633 "aft",
10634 project_root,
10635 Some(expected_for_request),
10636 ),
10637 )
10638 .await
10639 .unwrap()
10640 });
10641 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10642 .await
10643 .unwrap()
10644 .unwrap();
10645 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10646 let ModuleControlRequest::RouteBind {
10647 consumer_capabilities,
10648 ..
10649 } = bind
10650 else {
10651 panic!("expected route.bind request, got {bind:?}");
10652 };
10653 assert_eq!(consumer_capabilities, Some(expected.clone()));
10654
10655 handler
10656 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10657 .await
10658 .unwrap();
10659 let route_response = route_task.await.unwrap();
10660 assert!(route_response.is_empty());
10661 let published = client_rx.recv().await.unwrap();
10662 assert!(matches!(
10663 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10664 ClientControlResponse::RouteOpen { .. }
10665 ));
10666 }
10667
10668 #[tokio::test]
10669 async fn route_open_without_consumer_capabilities_relays_none() {
10670 let registry = Arc::new(Registry::default());
10671 let forwarding = Arc::new(ForwardingTable::default());
10672 let handler =
10673 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10674 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
10675 hello_via_sink(
10676 &handler,
10677 &module_ctx,
10678 &mut module_rx,
10679 hello_frame("aft", PROTOCOL_VERSION, 7),
10680 )
10681 .await;
10682
10683 let project_root = unique_project_root("consumer-capabilities-absent");
10684 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
10685 let route_handler = handler.clone();
10686 let route_task = tokio::spawn(async move {
10687 route_handler
10688 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
10689 .await
10690 .unwrap()
10691 });
10692 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10693 .await
10694 .unwrap()
10695 .unwrap();
10696 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10697 let ModuleControlRequest::RouteBind {
10698 consumer_capabilities,
10699 ..
10700 } = bind
10701 else {
10702 panic!("expected route.bind request, got {bind:?}");
10703 };
10704 assert_eq!(consumer_capabilities, None);
10705
10706 handler
10707 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10708 .await
10709 .unwrap();
10710 let route_response = route_task.await.unwrap();
10711 assert!(route_response.is_empty());
10712 let published = client_rx.recv().await.unwrap();
10713 assert!(matches!(
10714 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10715 ClientControlResponse::RouteOpen { .. }
10716 ));
10717 }
10718
10719 async fn bind_role_versions_for(
10723 sent: Option<BTreeMap<String, String>>,
10724 connection: u64,
10725 ) -> Option<BTreeMap<String, String>> {
10726 let registry = Arc::new(Registry::default());
10727 let forwarding = Arc::new(ForwardingTable::default());
10728 let handler =
10729 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10730 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
10731 hello_via_sink(
10732 &handler,
10733 &module_ctx,
10734 &mut module_rx,
10735 hello_frame("aft", PROTOCOL_VERSION, 7),
10736 )
10737 .await;
10738 let project_root = unique_project_root("role-versions");
10739 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
10740 let route_handler = handler.clone();
10741 let route_task = tokio::spawn(async move {
10742 route_handler
10743 .handle_control_frame(
10744 &client_ctx,
10745 route_open_frame_with_role_versions(403, "aft", project_root, sent),
10746 )
10747 .await
10748 .unwrap()
10749 });
10750 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10751 .await
10752 .expect("a well-formed route.open reaches the module as a bind")
10753 .unwrap();
10754 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10755 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
10756 panic!("expected route.bind request, got {bind:?}");
10757 };
10758 handler
10759 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10760 .await
10761 .unwrap();
10762 assert!(route_task.await.unwrap().is_empty());
10763 let published = client_rx.recv().await.unwrap();
10764 assert!(matches!(
10765 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10766 ClientControlResponse::RouteOpen { .. }
10767 ));
10768 role_versions
10769 }
10770
10771 #[tokio::test]
10772 async fn route_open_relays_role_versions_verbatim() {
10773 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
10774 assert_eq!(
10775 bind_role_versions_for(Some(sent.clone()), 141).await,
10776 Some(sent)
10777 );
10778 }
10779
10780 #[tokio::test]
10783 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
10784 assert_eq!(bind_role_versions_for(None, 143).await, None);
10785 assert_eq!(
10786 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
10787 None
10788 );
10789 }
10790
10791 #[tokio::test]
10792 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
10793 let registry = Arc::new(Registry::default());
10794 let forwarding = Arc::new(ForwardingTable::default());
10795 let handler =
10796 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10797 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
10798 hello_via_sink(
10799 &handler,
10800 &module_ctx,
10801 &mut module_rx,
10802 hello_frame("aft", PROTOCOL_VERSION, 7),
10803 )
10804 .await;
10805
10806 let nine: BTreeMap<String, String> = (0..9)
10807 .map(|index| (format!("role-{index}"), "v1".to_string()))
10808 .collect();
10809 for (label, malformed) in [
10810 (
10811 "invalid role name",
10812 role_versions(&[("Tool_Provider", "v1")]),
10813 ),
10814 ("invalid version", role_versions(&[("tool-provider", "v0")])),
10815 ("nine entries", nine),
10816 ] {
10817 let responses = tokio::time::timeout(
10820 Duration::from_secs(1),
10821 handler.handle_control_frame(
10822 &route_ctx(ConnectionId::new(148)).0,
10823 route_open_frame_with_role_versions(
10824 404,
10825 "aft",
10826 unique_project_root("role-versions-malformed"),
10827 Some(malformed),
10828 ),
10829 ),
10830 )
10831 .await
10832 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
10833 .unwrap();
10834 assert_eq!(responses.len(), 1, "{label}");
10835 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
10836 let error = parse_error(&responses[0]);
10837 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
10838 assert_eq!(
10839 error["detail"]["field"], "role_versions",
10840 "{label}: {error}"
10841 );
10842 assert!(
10843 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
10844 "{label}: a malformed declaration is terminal"
10845 );
10846 assert!(
10847 module_rx.try_recv().is_err(),
10848 "{label}: the module must never see a bind"
10849 );
10850 }
10851 }
10852
10853 #[tokio::test]
10857 async fn route_role_versions_capability_is_advertised() {
10858 let handler = ControlHandler::new(Arc::new(Registry::default()));
10859 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
10860 let ack = hello_via_sink(
10861 &handler,
10862 &ctx,
10863 &mut rx,
10864 hello_frame("m", PROTOCOL_VERSION, 1),
10865 )
10866 .await;
10867 let ack = parse_ack(&ack);
10868 assert!(
10869 ack.subc_capabilities
10870 .iter()
10871 .any(|c| c == "route-role-versions/v1"),
10872 "{:?}",
10873 ack.subc_capabilities
10874 );
10875
10876 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
10877 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
10878 let reply = handler
10879 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
10880 .await
10881 .unwrap()
10882 .pop()
10883 .unwrap();
10884 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10885 serde_json::from_slice(&reply.body).unwrap()
10886 else {
10887 panic!("not a server.describe reply");
10888 };
10889 assert!(
10890 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
10891 "{capabilities:?}"
10892 );
10893 }
10894
10895 #[tokio::test]
10899 async fn catalog_events_capability_is_advertised() {
10900 let handler = ControlHandler::new(Arc::new(Registry::default()));
10901 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
10902 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
10903 let reply = handler
10904 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
10905 .await
10906 .unwrap()
10907 .pop()
10908 .unwrap();
10909 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10910 serde_json::from_slice(&reply.body).unwrap()
10911 else {
10912 panic!("not a server.describe reply");
10913 };
10914 assert!(
10915 capabilities
10916 .iter()
10917 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
10918 "{capabilities:?}"
10919 );
10920 }
10921
10922 #[tokio::test]
10923 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
10924 let registry = Arc::new(Registry::default());
10925 let forwarding = Arc::new(ForwardingTable::default());
10926 let handler =
10927 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10928 .with_health_probe_timeout(Duration::from_secs(5));
10929 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
10930 hello_via_sink(
10931 &handler,
10932 &module_ctx,
10933 &mut module_rx,
10934 non_routable_hello_frame_with_control_ops(
10935 "mcp",
10936 300,
10937 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10938 ),
10939 )
10940 .await;
10941 assert!(registry
10942 .get_module("mcp")
10943 .unwrap()
10944 .unwrap()
10945 .manifest
10946 .provides
10947 .is_empty());
10948
10949 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
10950 let route_response = handler
10951 .handle_control_frame(
10952 &route_client_ctx,
10953 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
10954 )
10955 .await
10956 .unwrap();
10957 assert_eq!(route_response[0].header.ty, FrameType::Error);
10958 assert_eq!(
10959 parse_error(&route_response[0])["code"],
10960 "target_unavailable"
10961 );
10962 assert!(parse_error(&route_response[0])["message"]
10963 .as_str()
10964 .unwrap()
10965 .contains("does not provide the requested target"));
10966 assert!(module_rx.try_recv().is_err());
10967
10968 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
10969 let health_handler = handler.clone();
10970 let health_task = tokio::spawn(async move {
10971 health_handler
10972 .handle_control_frame(
10973 &health_client_ctx,
10974 supervisor_health_probe_frame(302, "mcp"),
10975 )
10976 .await
10977 .unwrap()
10978 });
10979 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10980 .await
10981 .unwrap()
10982 .unwrap();
10983 assert_eq!(
10984 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10985 ModuleControlRequest::HealthCheck {}
10986 );
10987 handler
10988 .handle_control_frame(
10989 &module_ctx,
10990 health_response(health_frame.header.corr, HealthStatus::Ok),
10991 )
10992 .await
10993 .unwrap();
10994 let health_response = health_task.await.unwrap();
10995 assert_eq!(health_response[0].header.ty, FrameType::Response);
10996 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10997 ClientControlResponse::SupervisorHealthProbe {
10998 module_id, status, ..
10999 } => {
11000 assert_eq!(module_id, "mcp");
11001 assert_eq!(status, HealthStatus::Ok);
11002 }
11003 other => panic!("unexpected health response: {other:?}"),
11004 }
11005
11006 forwarding
11011 .cleanup_connection(module_ctx.connection_id)
11012 .unwrap();
11013 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
11014 let cleanup_response = tokio::time::timeout(
11015 Duration::from_millis(200),
11016 handler.handle_control_frame(
11017 &cleanup_probe_ctx,
11018 supervisor_health_probe_frame(303, "mcp"),
11019 ),
11020 )
11021 .await
11022 .expect("probe should fail immediately when the control lane is gone")
11023 .unwrap();
11024 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
11025 assert_eq!(
11026 parse_error(&cleanup_response[0])["code"],
11027 "target_unavailable"
11028 );
11029 assert!(parse_error(&cleanup_response[0])["message"]
11030 .as_str()
11031 .unwrap()
11032 .contains("no module connection"));
11033
11034 handler
11035 .cleanup_connection(module_ctx.connection_id)
11036 .unwrap();
11037 }
11038
11039 #[tokio::test]
11040 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
11041 let registry = Arc::new(Registry::default());
11042 let supervisor_handle = SupervisorHandle::new();
11043 let supervisor =
11044 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11045 .with_handle(supervisor_handle.clone())
11046 .with_connection_file_path(
11047 std::env::temp_dir()
11048 .join(format!("subc-route-open-warming-{}", std::process::id())),
11049 );
11050 let module = supervisor
11051 .supervise_configured(
11052 ModuleSpec {
11053 module_id: "warming".to_string(),
11054 program: fake_aft_stub_path(),
11055 args: Vec::new(),
11056 env: Vec::new(),
11057 reserved: false,
11058 reserved_prefixes: Vec::new(),
11059 protocol: ModuleProtocol::Subc,
11060 overlap: Default::default(),
11061 },
11062 true,
11063 )
11064 .unwrap();
11065 assert_eq!(module.state().unwrap(), ModuleState::Running);
11066
11067 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11068 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
11069 let response = handler
11070 .handle_control_frame(
11071 &ctx,
11072 route_open_frame(304, "warming", unique_project_root("warming")),
11073 )
11074 .await
11075 .unwrap();
11076 module.stop().await.unwrap();
11077
11078 assert_eq!(response[0].header.ty, FrameType::Error);
11079 let error = parse_error(&response[0]);
11080 assert_eq!(error["code"], "module_warming");
11081 assert!(error["message"]
11082 .as_str()
11083 .unwrap()
11084 .contains("state=running, enabled=true, live=false"));
11085 }
11086
11087 #[test]
11088 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
11089 let handler = ControlHandler::new(Arc::new(Registry::default()));
11090 let capture = EventCapture::default();
11091 let _subscriber =
11092 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11093 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
11094 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
11095 let pending = (0..limit).collect::<Vec<_>>();
11096 let response = handler
11097 .route_open_capacity_refusal(
11098 &ctx,
11099 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
11100 "busy",
11101 pending.len(),
11102 limit,
11103 )
11104 .unwrap();
11105 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11106 let event = capture
11107 .events()
11108 .into_iter()
11109 .find(|event| {
11110 event.target == "control"
11111 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
11112 })
11113 .expect("connection admission refusal event");
11114 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11115 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11116 }
11117
11118 #[test]
11119 fn route_open_target_cap_logs_admission_reason_and_capacity() {
11120 let handler = ControlHandler::new(Arc::new(Registry::default()));
11121 let capture = EventCapture::default();
11122 let _subscriber =
11123 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11124 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
11125 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
11126 let guards = (0..limit)
11127 .map(|_| {
11128 handler
11129 .route_bind_concurrency
11130 .try_admit("busy", limit)
11131 .unwrap()
11132 })
11133 .collect::<Vec<_>>();
11134 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
11135 Err(in_flight) => in_flight,
11136 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
11137 };
11138 let response = handler
11139 .route_open_target_capacity_refusal(
11140 &ctx,
11141 &route_open_frame(397, "busy", unique_project_root("target-cap")),
11142 "busy",
11143 in_flight,
11144 )
11145 .unwrap();
11146 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11147 let event = capture
11148 .events()
11149 .into_iter()
11150 .find(|event| {
11151 event.target == "control"
11152 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
11153 })
11154 .expect("target admission refusal event");
11155 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11156 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11157 drop(guards);
11158 }
11159
11160 #[tokio::test(flavor = "current_thread")]
11161 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
11162 let handler = ControlHandler::new(Arc::new(Registry::default()));
11163 let capture = EventCapture::default();
11164 let _subscriber =
11165 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11166 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
11167 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11168
11169 let _ = handler
11170 .handle_client_control_request(
11171 &ctx,
11172 frame,
11173 ClientControlRequest::SupervisorSetEnabled {
11174 module_id: "broca".to_string(),
11175 enabled: false,
11176 },
11177 )
11178 .await
11179 .unwrap();
11180
11181 let events = capture
11182 .events()
11183 .into_iter()
11184 .filter(|event| {
11185 event.target == "control"
11186 && event.fields.get("message").map(String::as_str)
11187 == Some("supervisor request received")
11188 })
11189 .collect::<Vec<_>>();
11190 assert_eq!(events.len(), 1, "one supervisor request log line");
11191 let event = &events[0];
11192 assert_eq!(event.level, tracing::Level::INFO);
11193 assert_eq!(
11194 event.fields.get("op"),
11195 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
11196 );
11197 assert_eq!(
11198 event.fields.get("module_id"),
11199 Some(&"\"broca\"".to_string())
11200 );
11201 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
11202 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
11203 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
11204 }
11205
11206 #[tokio::test(flavor = "current_thread")]
11207 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
11208 let registry = Arc::new(Registry::default());
11209 let module_connection = ConnectionId::new(303);
11210 registry
11211 .register_with_control_ops(
11212 manifest("aft", PROTOCOL_VERSION),
11213 PROTOCOL_VERSION,
11214 module_connection,
11215 module_baseline_control_ops(),
11216 )
11217 .unwrap();
11218 let handler = ControlHandler::new(registry);
11219 let capture = EventCapture::default();
11220 let _subscriber =
11221 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11222 let (ctx, _rx) = route_ctx(module_connection);
11223 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11224
11225 let _ = handler
11226 .handle_client_control_request(
11227 &ctx,
11228 frame,
11229 ClientControlRequest::SupervisorSetEnabled {
11230 module_id: "broca".to_string(),
11231 enabled: false,
11232 },
11233 )
11234 .await
11235 .unwrap();
11236
11237 let callers = capture
11238 .events()
11239 .into_iter()
11240 .filter(|event| {
11241 event.fields.get("message").map(String::as_str)
11242 == Some("supervisor request received")
11243 })
11244 .map(|event| event.fields.get("caller").cloned())
11245 .collect::<Vec<_>>();
11246 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
11247 }
11248
11249 #[tokio::test(flavor = "current_thread")]
11250 async fn supervisor_list_does_not_log_request_received() {
11251 let handler = ControlHandler::new(Arc::new(Registry::default()));
11252 let capture = EventCapture::default();
11253 let _subscriber =
11254 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11255 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
11256 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
11257
11258 handler
11259 .handle_client_control_request(&ctx, frame, ClientControlRequest::SupervisorList {})
11260 .await
11261 .unwrap();
11262
11263 assert!(capture.events().into_iter().all(|event| {
11264 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11265 }));
11266 }
11267
11268 #[tokio::test(flavor = "current_thread")]
11269 async fn supervisor_rescan_preview_does_not_log_request_received() {
11270 let handler = ControlHandler::new(Arc::new(Registry::default()));
11271 let capture = EventCapture::default();
11272 let _subscriber =
11273 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11274 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
11275 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
11276
11277 handler
11278 .handle_client_control_request(
11279 &ctx,
11280 frame,
11281 ClientControlRequest::SupervisorRescan { preview: true },
11282 )
11283 .await
11284 .unwrap();
11285
11286 assert!(capture.events().into_iter().all(|event| {
11287 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11288 }));
11289 }
11290
11291 #[tokio::test]
11296 async fn route_open_refusal_names_the_check_that_refused() {
11297 let handler = ControlHandler::new(Arc::new(Registry::default()));
11298 let capture = EventCapture::default();
11299 let _subscriber =
11300 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11301 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11302 let response = handler
11303 .handle_control_frame(
11304 &ctx,
11305 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
11306 )
11307 .await
11308 .unwrap();
11309
11310 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11311 let event = capture
11312 .events()
11313 .into_iter()
11314 .find(|event| {
11315 event.target == "control"
11316 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11317 })
11318 .expect("route.open refusal event");
11319 assert_eq!(
11320 event.fields.get("reason"),
11321 Some(&"\"not_registered\"".to_string())
11322 );
11323 }
11324
11325 #[tokio::test]
11326 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
11327 let registry = Arc::new(Registry::default());
11328 let supervisor_handle = SupervisorHandle::new();
11329 let supervisor =
11330 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11331 .with_handle(supervisor_handle.clone())
11332 .with_connection_file_path(std::env::temp_dir().join(format!(
11333 "subc-route-open-refusal-info-{}",
11334 std::process::id()
11335 )));
11336 let module = supervisor
11337 .supervise_configured(
11338 ModuleSpec {
11339 module_id: "warming".to_string(),
11340 program: fake_aft_stub_path(),
11341 args: Vec::new(),
11342 env: Vec::new(),
11343 reserved: false,
11344 reserved_prefixes: Vec::new(),
11345 protocol: ModuleProtocol::Subc,
11346 overlap: Default::default(),
11347 },
11348 true,
11349 )
11350 .unwrap();
11351 assert_eq!(module.state().unwrap(), ModuleState::Running);
11352
11353 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11354 assert!(handler
11355 .counters()
11356 .snapshot()
11357 .get("route_open_refused_by_code")
11358 .is_none());
11359 let capture = EventCapture::default();
11360 let _subscriber =
11361 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11362 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
11363 let response = handler
11364 .handle_control_frame(
11365 &ctx,
11366 route_open_frame(394, "warming", unique_project_root("refusal-info")),
11367 )
11368 .await
11369 .unwrap();
11370 module.stop().await.unwrap();
11371
11372 assert_eq!(parse_error(&response[0])["code"], "module_warming");
11373 let event = capture
11374 .events()
11375 .into_iter()
11376 .find(|event| {
11377 event.target == "control"
11378 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
11379 })
11380 .expect("route.open refusal event");
11381 assert_eq!(
11382 event.fields.get("module_id"),
11383 Some(&"\"warming\"".to_string())
11384 );
11385 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
11386 assert_eq!(
11387 event.fields.get("reason"),
11388 Some(&"\"supervised_not_registered\"".to_string())
11389 );
11390 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
11391 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
11392 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
11393 assert_eq!(
11394 handler.counters().snapshot()["route_open_refused_by_code"],
11395 json!({ "module_warming": 1 })
11396 );
11397 }
11398
11399 const OUTAGE_START: &str = "route.open refusing module: not serving";
11400 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
11401
11402 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
11403 capture
11404 .events()
11405 .into_iter()
11406 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
11407 .collect()
11408 }
11409
11410 fn supervise_stub(
11411 registry: &Arc<Registry>,
11412 module_id: &str,
11413 enabled: bool,
11414 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
11415 let supervisor_handle = SupervisorHandle::new();
11416 let supervisor =
11417 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
11418 .with_handle(supervisor_handle.clone())
11419 .with_connection_file_path(std::env::temp_dir().join(format!(
11420 "subc-route-outage-{module_id}-{}",
11421 std::process::id()
11422 )));
11423 let module = supervisor
11424 .supervise_configured(
11425 ModuleSpec {
11426 module_id: module_id.to_string(),
11427 program: fake_aft_stub_path(),
11428 args: Vec::new(),
11429 env: Vec::new(),
11430 reserved: false,
11431 reserved_prefixes: Vec::new(),
11432 protocol: ModuleProtocol::Subc,
11433 overlap: Default::default(),
11434 },
11435 enabled,
11436 )
11437 .unwrap();
11438 (supervisor_handle, module)
11439 }
11440
11441 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
11442 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
11443 module_id: module_id.to_string(),
11444 drain_timeout_ms: Some(50),
11445 })
11446 .unwrap();
11447 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
11448 }
11449
11450 #[test]
11454 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
11455 let registry = Arc::new(Registry::default());
11456 let forwarding = Arc::new(ForwardingTable::default());
11457 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11458 let second = ControlHandler::with_forwarding(registry, forwarding);
11459 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
11460 }
11461
11462 #[tokio::test(flavor = "current_thread")]
11466 async fn route_open_unknown_module_refusals_add_no_outage_state() {
11467 let handler = ControlHandler::new(Arc::new(Registry::default()));
11468 let capture = EventCapture::default();
11469 let _subscriber =
11470 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11471 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
11472 for corr in 0..8 {
11473 let response = handler
11474 .handle_control_frame(
11475 &ctx,
11476 route_open_frame(
11477 380 + corr,
11478 &format!("nobody-{corr}"),
11479 unique_project_root("outage-unknown"),
11480 ),
11481 )
11482 .await
11483 .unwrap();
11484 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11485 }
11486
11487 assert_eq!(handler.route_outages.tracked_module_count(), 0);
11488 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
11489 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
11490 }
11491
11492 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11497 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
11498 let registry = Arc::new(Registry::default());
11499 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
11500 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11501 let capture = EventCapture::default();
11502 let _subscriber =
11503 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11504 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
11505 handler.route_outages.record_accepted("outage-restart");
11508
11509 let response = handler
11510 .handle_control_frame(
11511 &ctx,
11512 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
11513 )
11514 .await
11515 .unwrap();
11516 assert_eq!(response[0].header.ty, FrameType::Error);
11517 let starts = outage_lines(&capture, OUTAGE_START);
11518 assert_eq!(starts.len(), 1, "{starts:?}");
11519 assert_eq!(starts[0].level, tracing::Level::WARN);
11520 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
11521 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
11522 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
11523 handler.route_outages.record_accepted("outage-restart");
11524 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
11525
11526 let restart = handler
11527 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
11528 .await
11529 .unwrap();
11530 assert_eq!(
11531 restart[0].header.ty,
11532 FrameType::Response,
11533 "{:?}",
11534 parse_error(&restart[0])
11535 );
11536 handler
11537 .handle_control_frame(
11538 &ctx,
11539 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
11540 )
11541 .await
11542 .unwrap();
11543 module.stop().await.unwrap();
11544
11545 let starts = outage_lines(&capture, OUTAGE_START);
11546 assert_eq!(starts.len(), 2, "{starts:?}");
11547 assert_eq!(starts[1].level, tracing::Level::INFO);
11548 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
11549 }
11550
11551 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
11555 async fn failed_operator_restart_leaves_no_operator_mark() {
11556 let registry = Arc::new(Registry::default());
11557 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
11558 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11559 let capture = EventCapture::default();
11560 let _subscriber =
11561 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11562 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
11563 handler.route_outages.record_accepted("outage-disabled");
11564
11565 let restart = handler
11566 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
11567 .await
11568 .unwrap();
11569 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
11570 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
11571
11572 handler
11573 .handle_control_frame(
11574 &ctx,
11575 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
11576 )
11577 .await
11578 .unwrap();
11579 let starts = outage_lines(&capture, OUTAGE_START);
11580 assert_eq!(starts.len(), 1, "{starts:?}");
11581 assert_eq!(starts[0].level, tracing::Level::WARN);
11582 }
11583
11584 #[tokio::test(flavor = "current_thread")]
11585 async fn route_open_unknown_module_escapes_target_module_id() {
11586 let handler = ControlHandler::new(Arc::new(Registry::default()));
11587 let capture = EventCapture::default();
11588 let _subscriber =
11589 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11590 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
11591 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11592 let response = handler
11593 .handle_control_frame(
11594 &ctx,
11595 route_open_frame(
11596 395,
11597 hostile_module_id,
11598 unique_project_root("hostile-target-module-id"),
11599 ),
11600 )
11601 .await
11602 .unwrap();
11603
11604 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11605 let event = capture
11606 .events()
11607 .into_iter()
11608 .find(|event| {
11609 event.target == "control"
11610 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11611 })
11612 .expect("route.open unknown-module refusal event");
11613 let logged = event.fields.get("module_id").expect("module_id field");
11614 assert!(!logged.bytes().any(|byte| byte < 0x20));
11615 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
11616 }
11617
11618 #[tokio::test(flavor = "current_thread")]
11619 async fn route_open_module_rejection_uses_daemon_counter_key() {
11620 let registry = Arc::new(Registry::default());
11621 let forwarding = Arc::new(ForwardingTable::default());
11622 let handler =
11623 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11624 let module_connection = ConnectionId::new(95);
11625 let (module_ctx, mut module_rx) = route_ctx(module_connection);
11626 hello_via_sink(
11627 &handler,
11628 &module_ctx,
11629 &mut module_rx,
11630 hello_frame("aft", PROTOCOL_VERSION, 395),
11631 )
11632 .await;
11633
11634 let client_connection = ConnectionId::new(96);
11635 let (client_ctx, _client_rx) = route_ctx(client_connection);
11636 let capture = EventCapture::default();
11637 let _subscriber =
11638 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11639 let (route_task, bind) = relay_route_open(
11640 &handler,
11641 client_connection,
11642 &client_ctx.egress,
11643 &mut module_rx,
11644 396,
11645 "aft",
11646 "hostile-module-code",
11647 )
11648 .await;
11649 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
11650 let rejection = Frame::build(
11651 FrameType::Error,
11652 control_flags(),
11653 0,
11654 0,
11655 bind.header.corr,
11656 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
11657 )
11658 .unwrap();
11659 handler
11660 .handle_control_frame(&module_ctx, rejection)
11661 .await
11662 .unwrap();
11663
11664 let response = route_task.await.unwrap();
11665 assert_eq!(parse_error(&response[0])["code"], hostile_code);
11666 let counters = handler.counters().snapshot();
11667 assert_eq!(
11668 counters["route_open_refused_by_code"],
11669 json!({ "module_rejected": 1 })
11670 );
11671 assert!(counters["route_open_refused_by_code"]
11672 .get(hostile_code)
11673 .is_none());
11674
11675 let event = capture
11676 .events()
11677 .into_iter()
11678 .find(|event| {
11679 event.target == "control"
11680 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
11681 })
11682 .expect("route.open module-rejection refusal event");
11683 let logged = event.fields.get("module_code").expect("module_code field");
11684 assert!(!logged.bytes().any(|byte| byte < 0x20));
11685 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
11686 }
11687
11688 #[tokio::test]
11689 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
11690 let registry = Arc::new(Registry::default());
11691 let supervisor_handle = SupervisorHandle::new();
11692 let missing_program = std::env::temp_dir().join(format!(
11693 "subc-route-open-missing-program-{}",
11694 std::process::id()
11695 ));
11696 let supervisor =
11697 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11698 .with_handle(supervisor_handle.clone());
11699 let module = supervisor
11700 .supervise_configured(
11701 ModuleSpec {
11702 module_id: "failed".to_string(),
11703 program: missing_program,
11704 args: Vec::new(),
11705 env: Vec::new(),
11706 reserved: false,
11707 reserved_prefixes: Vec::new(),
11708 protocol: ModuleProtocol::Subc,
11709 overlap: Default::default(),
11710 },
11711 true,
11712 )
11713 .unwrap();
11714 assert_eq!(module.state().unwrap(), ModuleState::Failed);
11715
11716 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11717 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
11718 let response = handler
11719 .handle_control_frame(
11720 &ctx,
11721 route_open_frame(305, "failed", unique_project_root("failed")),
11722 )
11723 .await
11724 .unwrap();
11725
11726 assert_eq!(response[0].header.ty, FrameType::Error);
11727 let error = parse_error(&response[0]);
11728 assert_eq!(error["code"], "target_unavailable");
11729 assert!(error["message"]
11730 .as_str()
11731 .unwrap()
11732 .contains("state=failed, enabled=true, live=false"));
11733 }
11734
11735 #[tokio::test]
11736 async fn route_open_role_mismatch_remains_target_unavailable() {
11737 let registry = Arc::new(Registry::default());
11738 let handler = ControlHandler::new(Arc::clone(®istry));
11739 handler
11740 .handle_control(
11741 ConnectionId::new(41),
11742 non_routable_hello_frame_with_control_ops("health-only", 306, None),
11743 )
11744 .unwrap();
11745
11746 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
11747 let response = handler
11748 .handle_control_frame(
11749 &ctx,
11750 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
11751 )
11752 .await
11753 .unwrap();
11754
11755 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11756 assert!(parse_error(&response[0])["message"]
11757 .as_str()
11758 .unwrap()
11759 .contains("does not provide the requested target"));
11760 }
11761
11762 #[tokio::test]
11763 async fn route_open_inactive_registration_remains_target_unavailable() {
11764 let registry = Arc::new(Registry::default());
11765 let handler = ControlHandler::new(Arc::clone(®istry));
11766 handler
11767 .handle_control(
11768 ConnectionId::new(43),
11769 hello_frame("inactive", PROTOCOL_VERSION, 308),
11770 )
11771 .unwrap();
11772 assert!(registry
11773 .set_module_state_for_test("inactive", ChannelState::Closed)
11774 .unwrap());
11775
11776 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
11777 let response = handler
11778 .handle_control_frame(
11779 &ctx,
11780 route_open_frame(309, "inactive", unique_project_root("inactive")),
11781 )
11782 .await
11783 .unwrap();
11784
11785 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11786 assert!(parse_error(&response[0])["message"]
11787 .as_str()
11788 .unwrap()
11789 .contains("is not active"));
11790 }
11791
11792 #[tokio::test]
11793 async fn late_health_reply_is_recorded_through_the_module_response_path() {
11794 let registry = Arc::new(Registry::default());
11795 let forwarding = Arc::new(ForwardingTable::default());
11796 let supervisor_handle = SupervisorHandle::new();
11797 let supervisor =
11798 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
11799 .with_forwarding(Arc::clone(&forwarding))
11800 .with_handle(supervisor_handle.clone());
11801 let module = supervisor
11802 .supervise_configured(
11803 crate::ModuleSpec {
11804 module_id: "late-health-response".to_string(),
11805 program: PathBuf::from("disabled-module"),
11806 args: Vec::new(),
11807 env: Vec::new(),
11808 reserved: false,
11809 reserved_prefixes: Vec::new(),
11810 protocol: ModuleProtocol::Subc,
11811 overlap: Default::default(),
11812 },
11813 false,
11814 )
11815 .unwrap();
11816 let handler =
11817 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11818 .with_supervisor(supervisor_handle);
11819 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
11820 handler
11821 .handle_control_frame(
11822 &module_ctx,
11823 hello_frame_with_control_ops(
11824 "late-health-response",
11825 PROTOCOL_VERSION,
11826 7,
11827 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11828 ),
11829 )
11830 .await
11831 .unwrap();
11832 let probe_started_at = Instant::now() - Duration::from_millis(80);
11833 let pending = forwarding
11834 .begin_health_probe_rpc_for(
11835 "late-health-response",
11836 MODULE_CONTROL_OP_HEALTH_CHECK,
11837 probe_started_at,
11838 Instant::now() - Duration::from_millis(1),
11839 )
11840 .unwrap();
11841 assert!(forwarding
11842 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
11843 .unwrap());
11844
11845 let responses = handler
11846 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
11847 .await
11848 .unwrap();
11849
11850 assert!(responses.is_empty());
11851 let health = module.status().unwrap().health;
11852 assert_eq!(health.late_answer_count, 1);
11853 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
11854 }
11855
11856 #[tokio::test]
11857 async fn health_probe_timeout_and_module_death_are_typed() {
11858 let registry = Arc::new(Registry::default());
11859 let forwarding = Arc::new(ForwardingTable::default());
11860 let handler =
11861 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11862 .with_health_probe_timeout(Duration::from_millis(50));
11863 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11864 hello_via_sink(
11865 &handler,
11866 &module_ctx,
11867 &mut module_rx,
11868 hello_frame_with_control_ops(
11869 "aft",
11870 PROTOCOL_VERSION,
11871 7,
11872 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11873 ),
11874 )
11875 .await;
11876
11877 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11878 let responses = handler
11879 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
11880 .await
11881 .unwrap();
11882 assert_eq!(responses[0].header.ty, FrameType::Error);
11883 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
11884 let _ = module_rx.try_recv();
11885
11886 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
11887 let health_handler = handler.clone();
11888 let death_task = tokio::spawn(async move {
11889 health_handler
11890 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
11891 .await
11892 .unwrap()
11893 });
11894 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11895 .await
11896 .unwrap()
11897 .unwrap();
11898 handler
11899 .cleanup_connection(module_ctx.connection_id)
11900 .unwrap();
11901 let responses = death_task.await.unwrap();
11902 assert_eq!(responses[0].header.ty, FrameType::Error);
11903 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
11904 }
11905
11906 #[test]
11907 fn hello_requires_exact_protocol_version() {
11908 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
11909 let registry = Arc::new(Registry::default());
11910 let handler = ControlHandler::new(Arc::clone(®istry));
11911 let responses = handler
11912 .handle_control(
11913 ConnectionId::new(connection),
11914 hello_frame("aft", offered, 9),
11915 )
11916 .unwrap();
11917
11918 assert_eq!(responses.len(), 1);
11919 assert_eq!(responses[0].header.ty, FrameType::Error);
11920 let error = parse_error(&responses[0]);
11921 assert_eq!(error["code"], "version_unsupported");
11922 assert!(registry.get_module("aft").unwrap().is_none());
11923 assert_eq!(registry.active_registration_count().unwrap(), 0);
11924 }
11925 }
11926
11927 #[test]
11928 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
11929 let registry = Arc::new(Registry::default());
11930 let forwarding = Arc::new(ForwardingTable::default());
11931 let handler =
11932 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11933 let module_connection = ConnectionId::new(301);
11934 let registration = registry
11935 .register_with_control_ops(
11936 manifest("aft-push", PROTOCOL_VERSION),
11937 PROTOCOL_VERSION,
11938 module_connection,
11939 module_baseline_control_ops(),
11940 )
11941 .unwrap();
11942 let (module_tx, _module_rx) = mpsc::channel(8);
11943 let endpoint = forwarding
11944 .register_module_connection(
11945 module_connection,
11946 "aft-push".to_string(),
11947 PROTOCOL_VERSION,
11948 manifest_concurrency(®istration.manifest),
11949 FrameSink::new(module_tx),
11950 )
11951 .unwrap();
11952
11953 let unknown = Frame::build(
11955 FrameType::Push,
11956 control_flags(),
11957 0,
11958 0,
11959 5,
11960 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
11961 )
11962 .unwrap();
11963 let out = handler.handle_status_update(endpoint, unknown).unwrap();
11964 assert!(
11965 out.is_empty(),
11966 "unknown push op must be ignored, got {out:?}"
11967 );
11968
11969 let malformed = Frame::build(
11971 FrameType::Push,
11972 control_flags(),
11973 0,
11974 0,
11975 6,
11976 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
11977 )
11978 .unwrap();
11979 let out = handler.handle_status_update(endpoint, malformed).unwrap();
11980 assert_eq!(out.len(), 1);
11981 assert_eq!(out[0].header.ty, FrameType::Error);
11982 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
11983 }
11984
11985 #[test]
11986 fn hello_rejected_when_connection_already_owns_client_routes() {
11987 let registry = Arc::new(Registry::default());
11988 let forwarding = Arc::new(ForwardingTable::default());
11989 let handler =
11990 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11991 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
11993 let client_connection = ConnectionId::new(202);
11994
11995 let responses = handler
11998 .handle_control(
11999 client_connection,
12000 hello_frame("aft-second", PROTOCOL_VERSION, 9),
12001 )
12002 .unwrap();
12003 assert_eq!(responses[0].header.ty, FrameType::Error);
12004 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
12005 assert!(registry.get_module("aft-second").unwrap().is_none());
12006 }
12007
12008 #[tokio::test]
12009 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
12010 let registry = Arc::new(Registry::default());
12011 let forwarding = Arc::new(ForwardingTable::default());
12012 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
12013 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
12014 hello_via_sink(
12015 &handler,
12016 &module_ctx,
12017 &mut module_rx,
12018 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
12019 )
12020 .await;
12021 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
12022 let pending = forwarding
12023 .begin_route_bind_relay_for_test(
12024 client_ctx.connection_id,
12025 client_ctx.egress.clone(),
12026 2,
12027 "alpha",
12028 )
12029 .unwrap();
12030 forwarding
12031 .complete_pending_relay(
12032 module_ctx.connection_id,
12033 pending.corr,
12034 RouteBindRelayOutcome::Accepted,
12035 )
12036 .unwrap();
12037 client_rx.try_recv().unwrap();
12038 for module_id in ["beta", "alpha"] {
12039 let replies = handler
12040 .handle_control_frame(
12041 &module_ctx,
12042 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
12043 )
12044 .await
12045 .unwrap();
12046 assert_eq!(replies.len(), 1, "second HELLO must be refused");
12047 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
12048 }
12049 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
12050 assert!(registry.get_module("beta").unwrap().is_none());
12051 assert!(matches!(
12052 forwarding
12053 .lookup_data_route(
12054 client_ctx.connection_id,
12055 pending.client_channel,
12056 pending.client_epoch,
12057 )
12058 .unwrap(),
12059 DataRoute::Client(DataRouteState::Bound(_))
12060 ));
12061 assert!(handler
12062 .hello_launch_nonces
12063 .lock()
12064 .unwrap()
12065 .presented(module_ctx.connection_id, Some("alpha-nonce")));
12066 assert!(module_rx.try_recv().is_err());
12067 }
12068
12069 #[test]
12070 fn reserved_module_hello_requires_matching_launch_nonce() {
12071 let registry = Arc::new(Registry::default());
12072 let supervisor = SupervisorHandle::new();
12073 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12076 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12077
12078 let no_nonce = handler
12080 .handle_control(
12081 ConnectionId::new(1),
12082 hello_frame("vault", PROTOCOL_VERSION, 1),
12083 )
12084 .unwrap();
12085 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
12086 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
12087 assert!(registry.get_module("vault").unwrap().is_none());
12088
12089 let wrong = handler
12091 .handle_control(
12092 ConnectionId::new(2),
12093 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
12094 )
12095 .unwrap();
12096 assert_eq!(wrong[0].header.ty, FrameType::Error);
12097 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
12098 assert!(registry.get_module("vault").unwrap().is_none());
12099
12100 let ok = handler
12102 .handle_control(
12103 ConnectionId::new(3),
12104 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
12105 )
12106 .unwrap();
12107 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
12108 assert!(registry.get_module("vault").unwrap().is_some());
12109 }
12110
12111 #[test]
12112 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
12113 let registry = Arc::new(Registry::default());
12114 let supervisor = SupervisorHandle::new();
12115 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12116 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12117 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12118
12119 let squat = handler
12120 .handle_control(
12121 ConnectionId::new(1),
12122 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
12123 )
12124 .unwrap();
12125 assert_eq!(squat[0].header.ty, FrameType::Error);
12126 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
12127 assert!(parse_error(&squat[0])["message"]
12128 .as_str()
12129 .unwrap()
12130 .contains("fed:"));
12131
12132 let accepted_peer = handler
12133 .handle_control(
12134 ConnectionId::new(2),
12135 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
12136 )
12137 .unwrap();
12138 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
12139
12140 let accepted_short = handler
12141 .handle_control(
12142 ConnectionId::new(3),
12143 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
12144 )
12145 .unwrap();
12146 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
12147
12148 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
12149 let response = handler
12150 .handle_control(
12151 ConnectionId::new(conn),
12152 hello_frame(module_id, PROTOCOL_VERSION, conn),
12153 )
12154 .unwrap();
12155 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
12156 }
12157 }
12158
12159 #[test]
12160 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
12161 let registry = Arc::new(Registry::default());
12162 let supervisor = SupervisorHandle::new();
12163 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12164 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12165 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
12166 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12167
12168 let owner_nonce = handler
12169 .handle_control(
12170 ConnectionId::new(1),
12171 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
12172 )
12173 .unwrap();
12174 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
12175 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
12176 assert!(registry.get_module("fed:special").unwrap().is_none());
12177
12178 let exact_nonce = handler
12179 .handle_control(
12180 ConnectionId::new(2),
12181 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
12182 )
12183 .unwrap();
12184 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
12185 assert!(registry.get_module("fed:special").unwrap().is_some());
12186 }
12187
12188 #[test]
12189 fn non_reserved_module_ignores_launch_nonce() {
12190 let registry = Arc::new(Registry::default());
12191 let handler = ControlHandler::new(Arc::clone(®istry));
12194 let no_nonce = handler
12195 .handle_control(
12196 ConnectionId::new(1),
12197 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
12198 )
12199 .unwrap();
12200 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
12201 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
12202
12203 let echoed_nonce = handler
12204 .handle_control(
12205 ConnectionId::new(2),
12206 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
12207 )
12208 .unwrap();
12209 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
12210 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
12211 }
12212
12213 #[test]
12214 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
12215 let handler = ControlHandler::default();
12216 let conn = ConnectionId::new(1);
12217 let malformed = Frame::build(
12218 FrameType::Hello,
12219 control_flags(),
12220 0,
12221 0,
12222 3,
12223 b"{not json".to_vec(),
12224 )
12225 .unwrap();
12226
12227 let error = handler.handle_control(conn, malformed).unwrap();
12228 assert_eq!(error[0].header.ty, FrameType::Error);
12229 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
12230
12231 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12232 let pong = handler.handle_control(conn, ping).unwrap();
12233 assert_eq!(pong[0].header.ty, FrameType::Pong);
12234 assert_eq!(pong[0].header.corr, 4);
12235 }
12236
12237 #[test]
12238 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
12239 let registry = Arc::new(Registry::default());
12240 let handler = ControlHandler::new(Arc::clone(®istry));
12241
12242 handler
12243 .handle_control(
12244 ConnectionId::new(1),
12245 hello_frame("aft", PROTOCOL_VERSION, 1),
12246 )
12247 .unwrap();
12248 let duplicate = handler
12249 .handle_control(
12250 ConnectionId::new(2),
12251 hello_frame("aft", PROTOCOL_VERSION, 2),
12252 )
12253 .unwrap();
12254
12255 assert_eq!(duplicate[0].header.ty, FrameType::Error);
12256 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12257 let registration = registry.get_module("aft").unwrap().unwrap();
12258 assert_eq!(registration.connection_id, ConnectionId::new(1));
12259 }
12260
12261 #[test]
12262 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
12263 let registry = Arc::new(Registry::default());
12264 let forwarding = Arc::new(ForwardingTable::default());
12265 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
12266 let handler =
12267 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12268 .with_process_liveness(process_liveness);
12269 let (ctx, route_channel, route_epoch) =
12270 bind_liveness_route(®istry, &forwarding, "aft-dead");
12271 let responses = handler
12272 .handle_route_poll(
12273 &ctx,
12274 route_poll_frame(41, PollKind::Liveness, route_channel),
12275 route_channel,
12276 route_epoch,
12277 PollKind::Liveness,
12278 )
12279 .unwrap();
12280
12281 assert_eq!(responses.len(), 1);
12282 assert_eq!(responses[0].header.ty, FrameType::Response);
12283 assert_route_poll_liveness(&responses[0], false);
12284 }
12285
12286 #[test]
12287 fn liveness_poll_without_process_source_uses_bound_route() {
12288 let registry = Arc::new(Registry::default());
12289 let forwarding = Arc::new(ForwardingTable::default());
12290 let handler =
12291 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12292 let (ctx, route_channel, route_epoch) =
12293 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
12294 let responses = handler
12295 .handle_route_poll(
12296 &ctx,
12297 route_poll_frame(42, PollKind::Liveness, route_channel),
12298 route_channel,
12299 route_epoch,
12300 PollKind::Liveness,
12301 )
12302 .unwrap();
12303
12304 assert_route_poll_liveness(&responses[0], true);
12305 }
12306
12307 #[test]
12308 fn liveness_poll_untracked_process_source_uses_bound_route() {
12309 let registry = Arc::new(Registry::default());
12310 let forwarding = Arc::new(ForwardingTable::default());
12311 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
12312 let handler =
12313 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12314 .with_process_liveness(process_liveness);
12315 let (ctx, route_channel, route_epoch) =
12316 bind_liveness_route(®istry, &forwarding, "aft-untracked");
12317 let responses = handler
12318 .handle_route_poll(
12319 &ctx,
12320 route_poll_frame(43, PollKind::Liveness, route_channel),
12321 route_channel,
12322 route_epoch,
12323 PollKind::Liveness,
12324 )
12325 .unwrap();
12326
12327 assert_route_poll_liveness(&responses[0], true);
12328 }
12329
12330 #[tokio::test]
12331 async fn unknown_op_returns_unknown_control_op() {
12332 let handler = ControlHandler::default();
12333 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
12334 let request = Frame::build(
12335 FrameType::Request,
12336 control_flags(),
12337 0,
12338 0,
12339 55,
12340 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
12341 )
12342 .unwrap();
12343
12344 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12345
12346 assert_eq!(response.len(), 1);
12347 assert_eq!(response[0].header.ty, FrameType::Error);
12348 assert_eq!(response[0].header.corr, 55);
12349 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
12350 }
12351
12352 #[tokio::test]
12353 async fn supervisor_provenance_rejects_unknown_exact_module() {
12354 let handler = ControlHandler::default();
12355 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
12356 let request = Frame::build(
12357 FrameType::Request,
12358 control_flags(),
12359 0,
12360 0,
12361 57,
12362 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
12363 )
12364 .unwrap();
12365
12366 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12367
12368 assert_eq!(response.len(), 1);
12369 assert_eq!(response[0].header.ty, FrameType::Error);
12370 assert_eq!(response[0].header.corr, 57);
12371 let error = parse_error(&response[0]);
12372 assert_eq!(error["code"], "unknown_module");
12373 assert_eq!(error["message"], "module_id 'missing' is not supervised");
12374 }
12375
12376 #[test]
12377 fn provenance_probe_override_keeps_handler_tests_deterministic() {
12378 let expected = subc_control::RunningImageAgreement::Unavailable {
12379 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12380 };
12381 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
12382 assert_eq!(handler.provenance_probe_override, Some(expected));
12383 }
12384
12385 #[test]
12386 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
12387 let verdict = reload_verdict(
12388 std::path::Path::new("/bin/new"),
12389 Some(std::path::Path::new("/bin/old")),
12390 subc_control::RunningImageAgreement::Unavailable {
12391 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12392 },
12393 );
12394 assert!(matches!(
12395 verdict.path,
12396 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
12397 if configured == std::path::Path::new("/bin/new")
12398 && spawned_from == std::path::Path::new("/bin/old")
12399 ));
12400 }
12401
12402 #[cfg(windows)]
12403 #[test]
12404 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
12405 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
12406 let configured = dir.join("module.exe");
12407 std::fs::write(&configured, b"executable path fixture").unwrap();
12408 let spawned = std::fs::canonicalize(&configured).unwrap();
12409 assert_ne!(
12410 configured, spawned,
12411 "fixture must exercise different path spellings"
12412 );
12413 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
12414 let lower_case = configured.to_string_lossy().to_lowercase();
12415 for spelling in [
12416 &configured,
12417 Path::new(&forward_slashes),
12418 Path::new(&lower_case),
12419 ] {
12420 let verdict = reload_verdict(
12421 spelling,
12422 Some(&spawned),
12423 subc_control::RunningImageAgreement::Unavailable {
12424 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12425 },
12426 );
12427 assert_eq!(
12428 verdict.path,
12429 ReloadPathAgreement::Match,
12430 "configured {spelling:?}, spawned {spawned:?}"
12431 );
12432 }
12433 }
12434
12435 #[cfg(windows)]
12436 #[test]
12437 fn windows_reload_path_keeps_different_existing_program_pending() {
12438 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
12439 let old = dir.join("old.exe");
12440 let new = dir.join("new.exe");
12441 std::fs::write(&old, b"same executable content").unwrap();
12442 std::fs::copy(&old, &new).unwrap();
12443 let spawned = std::fs::canonicalize(&old).unwrap();
12444 let image = subc_control::RunningImageAgreement::Unavailable {
12445 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12446 };
12447 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
12448 assert!(
12449 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
12450 if configured == new && spawned_from == spawned)
12451 );
12452 let missing = dir.join("missing.exe");
12453 assert!(matches!(
12454 reload_verdict(&missing, Some(&spawned), image).path,
12455 ReloadPathAgreement::Mismatch { .. }
12456 ));
12457 }
12458
12459 #[test]
12460 fn reload_verdict_detects_replaced_image_at_same_path() {
12461 let image = subc_control::RunningImageAgreement::Mismatch {
12462 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
12463 digest: "old".into(),
12464 },
12465 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
12466 digest: "new".into(),
12467 },
12468 };
12469 let verdict = reload_verdict(
12470 std::path::Path::new("/bin/same"),
12471 Some(std::path::Path::new("/bin/same")),
12472 image.clone(),
12473 );
12474 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12475 assert_eq!(verdict.image, image);
12476 }
12477
12478 #[test]
12479 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
12480 let image = subc_control::RunningImageAgreement::Unavailable {
12481 reason: subc_control::RunningImageUnavailableReason::NotRunning,
12482 };
12483 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
12484 assert_eq!(
12485 verdict.path,
12486 subc_control::ReloadPathAgreement::Unavailable {
12487 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
12488 }
12489 );
12490 assert_eq!(verdict.image, image);
12491
12492 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
12493 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
12494 };
12495 let verdict = reload_verdict(
12496 std::path::Path::new("/bin/same"),
12497 Some(std::path::Path::new("/bin/same")),
12498 unconfirmed.clone(),
12499 );
12500 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12501 assert_eq!(verdict.image, unconfirmed);
12502 }
12503
12504 #[test]
12505 fn reload_verdict_preserves_each_image_unavailability_reason() {
12506 use subc_control::RunningImageUnavailableReason as Reason;
12507
12508 for reason in [
12509 Reason::NotRunning,
12510 Reason::UnsupportedPlatform,
12511 Reason::RunningExecutableUnreadable,
12512 Reason::SpawnedPathUnreadable,
12513 Reason::HashFailed,
12514 Reason::ProcessIdentityUnconfirmed,
12515 Reason::Unknown("future_probe_reason".to_string()),
12516 ] {
12517 let image = subc_control::RunningImageAgreement::Unavailable {
12518 reason: reason.clone(),
12519 };
12520 let verdict = reload_verdict(
12521 std::path::Path::new("/bin/same"),
12522 Some(std::path::Path::new("/bin/same")),
12523 image.clone(),
12524 );
12525 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
12526 assert_eq!(verdict.image, image, "{reason:?}");
12527 }
12528 }
12529
12530 #[tokio::test]
12531 async fn malformed_control_bodies_return_invalid_control_body() {
12532 let handler = ControlHandler::default();
12533 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
12534
12535 for (corr, body) in [
12536 (56, br#"{"route_channel":1}"#.as_slice()),
12537 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
12538 (
12539 58,
12540 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
12541 ),
12542 ] {
12543 let request = Frame::build(
12544 FrameType::Request,
12545 control_flags(),
12546 0,
12547 0,
12548 corr,
12549 body.to_vec(),
12550 )
12551 .unwrap();
12552 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12553
12554 assert_eq!(response.len(), 1);
12555 assert_eq!(response[0].header.ty, FrameType::Error);
12556 assert_eq!(response[0].header.corr, corr);
12557 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
12558 }
12559 }
12560
12561 #[tokio::test]
12562 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
12563 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12564 let registry = Arc::new(Registry::default());
12565 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
12566 let router = Router::with_control_handler(Arc::clone(&control));
12567 let connection = router.begin_connection();
12568 let (ctx, mut rx) = route_ctx(connection.id());
12569
12570 router
12571 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
12572 .await
12573 .unwrap();
12574 let response = rx.recv().await.unwrap();
12575 let ack = parse_ack(&response);
12576 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
12577 let channel = 1;
12578
12579 let goodbye =
12580 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
12581 router.route_for_connection(&ctx, goodbye).await.unwrap();
12582 assert!(rx.try_recv().is_err());
12583 assert!(registry.get_module("aft").unwrap().is_none());
12584
12585 router
12586 .route_for_connection(&ctx, channel_request(channel, 13))
12587 .await
12588 .unwrap();
12589 let error_frame = rx.recv().await.unwrap();
12590 assert_eq!(error_frame.header.ty, FrameType::Error);
12591 assert_eq!(error_frame.header.channel, channel);
12592 let captured = crate::router::test_log::captured_logs(&logs);
12593 assert_eq!(
12594 captured
12595 .lines()
12596 .filter(|line| {
12597 line.contains("module_id=aft")
12598 && line.contains("reason=explicit_goodbye")
12599 && line.contains("module registration ended")
12600 })
12601 .count(),
12602 1,
12603 "unexpected GOODBYE registry log: {captured}"
12604 );
12605 }
12606
12607 #[tokio::test]
12608 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
12609 let registry = Arc::new(Registry::default());
12610 let handler = ControlHandler::new(registry).with_capability_config(
12611 [("prov".to_string(), true), ("cons".to_string(), true)],
12612 BTreeMap::new(),
12613 );
12614 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
12615 register_capability_manifest(
12616 &handler,
12617 &provider_ctx,
12618 &mut provider_rx,
12619 capability_manifest("prov", &["thing/v1"], &[]),
12620 1,
12621 )
12622 .await;
12623 let mut consumer = capability_manifest("cons", &[], &[]);
12624 consumer.capabilities.as_mut().unwrap().requires.push(
12625 subc_protocol::manifest::CapabilityRequirement {
12626 capability: "thing/v1".to_string(),
12627 need: subc_protocol::manifest::CapabilityNeed::Required,
12628 },
12629 );
12630 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
12631 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
12632 assert_eq!(
12633 handler.capability_evaluator.verdict("cons", "thing/v1"),
12634 Some(CapabilityVerdict::Provided)
12635 );
12636 let (mut client_rx, _) = open_route_for_capability_test(
12637 &handler,
12638 &provider_ctx,
12639 &mut provider_rx,
12640 703,
12641 3,
12642 "prov",
12643 None,
12644 )
12645 .await;
12646 let goodbye =
12647 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12648 handler
12649 .handle_control_frame(&provider_ctx, goodbye)
12650 .await
12651 .unwrap();
12652 assert_eq!(
12653 handler.capability_evaluator.verdict("cons", "thing/v1"),
12654 Some(CapabilityVerdict::NeverProvided)
12655 );
12656 let closed = client_rx
12657 .try_recv()
12658 .expect("GOODBYE pushes route.closed before route GOODBYE");
12659 assert!(
12660 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
12661 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
12662 );
12663 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
12664 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
12665 }
12666
12667 struct RouteClosedFixture {
12668 handler: ControlHandler,
12669 #[cfg(unix)]
12670 supervisor: Supervisor,
12671 module_ctx: RouteCtx,
12672 #[cfg(unix)]
12673 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12674 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
12675 route: Arc<crate::forwarding::RouteBinding>,
12676 }
12677
12678 async fn route_closed_fixture() -> RouteClosedFixture {
12679 let registry = Arc::new(Registry::default());
12680 let handler = ControlHandler::new(Arc::clone(®istry));
12681 #[cfg(unix)]
12682 let supervisor = Supervisor::new(registry, RestartPolicy::default())
12683 .with_forwarding(Arc::clone(&handler.forwarding));
12684 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
12685 register_capability_manifest(
12686 &handler,
12687 &module_ctx,
12688 &mut module_rx,
12689 capability_manifest("prov", &[], &[]),
12690 1,
12691 )
12692 .await;
12693 let (client_rx, bind) = open_route_for_capability_test(
12694 &handler,
12695 &module_ctx,
12696 &mut module_rx,
12697 702,
12698 2,
12699 "prov",
12700 None,
12701 )
12702 .await;
12703 let ModuleControlRequest::RouteBind {
12704 route_channel: channel,
12705 epoch: route_epoch,
12706 ..
12707 } = bind
12708 else {
12709 panic!("expected route.bind");
12710 };
12711 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
12712 handler
12713 .forwarding
12714 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
12715 .unwrap()
12716 else {
12717 panic!("the acknowledged route must be bound");
12718 };
12719 RouteClosedFixture {
12720 handler,
12721 #[cfg(unix)]
12722 supervisor,
12723 module_ctx,
12724 #[cfg(unix)]
12725 module_rx,
12726 client_rx,
12727 route,
12728 }
12729 }
12730
12731 impl RouteClosedFixture {
12732 #[cfg(unix)]
12733 async fn cut(&mut self) {
12734 let notices = async {
12735 let mut draining = self.module_rx.recv().await.unwrap();
12736 assert!(matches!(
12737 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
12738 &draining.body
12739 )
12740 .unwrap(),
12741 subc_protocol::session::ModuleControlCommand::Draining {
12742 reason: RouteCloseReason::Restart,
12743 ..
12744 }
12745 ));
12746 draining.flushed.take().unwrap().send(()).unwrap();
12747 let mut closing = self.client_rx.recv().await.unwrap();
12748 assert!(matches!(
12749 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
12750 ClientControlPush::RouteClosing {
12751 reason: RouteCloseReason::Restart,
12752 ..
12753 }
12754 ));
12755 closing.flushed.take().unwrap().send(()).unwrap();
12756 };
12757 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
12758 result.unwrap();
12759 }
12760
12761 fn disconnect(&mut self) -> ClientControlPush {
12762 self.handler
12763 .cleanup_connection(self.module_ctx.connection_id)
12764 .unwrap();
12765 let closed = self.client_rx.try_recv().unwrap();
12766 assert_eq!(closed.header.ty, FrameType::Push);
12767 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
12768 assert!(matches!(
12769 &push,
12770 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
12771 if channels == &[self.route.client_channel]
12772 ));
12773 assert_eq!(
12774 self.client_rx.try_recv().unwrap().header.ty,
12775 FrameType::Goodbye
12776 );
12777 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
12778 push
12779 }
12780 }
12781
12782 #[cfg(unix)]
12783 #[tokio::test(start_paused = true)]
12784 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
12785 let mut fixture = route_closed_fixture().await;
12786 fixture.cut().await;
12787 assert!(matches!(
12788 fixture.disconnect(),
12789 ClientControlPush::RouteClosed {
12790 reason: RouteCloseReason::Restart,
12791 drained: true,
12792 ..
12793 }
12794 ));
12795 }
12796
12797 #[cfg(unix)]
12798 #[tokio::test(start_paused = true)]
12799 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
12800 let mut fixture = route_closed_fixture().await;
12801 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
12802 let started = tokio::time::Instant::now();
12803 fixture.cut().await;
12804 assert!(started.elapsed() >= Duration::from_secs(2));
12805 assert_eq!(fixture.route.flow.in_flight(), 1);
12806 assert!(matches!(
12807 fixture.disconnect(),
12808 ClientControlPush::RouteClosed {
12809 reason: RouteCloseReason::Restart,
12810 drained: false,
12811 ..
12812 }
12813 ));
12814 }
12815
12816 #[tokio::test]
12817 async fn module_connection_crash_reports_crash_and_not_drained() {
12818 let mut fixture = route_closed_fixture().await;
12819 assert!(matches!(
12823 fixture.disconnect(),
12824 ClientControlPush::RouteClosed {
12825 reason: RouteCloseReason::Crash,
12826 drained: false,
12827 ..
12828 }
12829 ));
12830 }
12831
12832 #[tokio::test]
12833 async fn dropping_router_connection_releases_registration() {
12834 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12835 let registry = Arc::new(Registry::default());
12836 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
12837 let router = Router::with_control_handler(Arc::clone(&control));
12838 let connection = router.begin_connection();
12839 let connection_id = connection.id();
12840 let (ctx, mut rx) = route_ctx(connection_id);
12841
12842 router
12843 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
12844 .await
12845 .unwrap();
12846 let response = rx.recv().await.unwrap();
12847 let ack = parse_ack(&response);
12848 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
12849 assert!(registry.get_module("aft").unwrap().is_some());
12850
12851 drop(connection);
12852
12853 assert!(registry.get_module("aft").unwrap().is_none());
12854 assert_eq!(registry.active_registration_count().unwrap(), 0);
12855
12856 control
12857 .cleanup_connection(ConnectionId::new(u64::MAX))
12858 .unwrap();
12859 let captured = crate::router::test_log::captured_logs(&logs);
12860 println!("captured registry lifecycle logs:\n{captured}");
12861 let events: Vec<_> = captured
12862 .lines()
12863 .filter(|line| {
12864 line.contains("module registered module_id=aft ")
12865 || line.contains("module registration ended")
12866 })
12867 .collect();
12868 assert_eq!(
12869 events.len(),
12870 2,
12871 "unexpected registry lifecycle logs: {captured}"
12872 );
12873 assert!(events[0].contains("module registered module_id=aft "));
12874 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
12875 assert!(events[1].contains(&format!(
12876 "module_id=aft connection_id={}",
12877 connection_id.get()
12878 )));
12879 assert!(events[1].contains("reason=connection_closed"));
12880 assert!(events[1].contains("module registration ended"));
12881 assert_eq!(
12882 captured
12883 .lines()
12884 .filter(|line| line.contains("module registered module_id=aft "))
12885 .count(),
12886 1,
12887 "legacy registration admission line must appear once: {captured}"
12888 );
12889 }
12890
12891 #[tokio::test]
12892 async fn hello_registration_keeps_legacy_module_registered_line_once() {
12893 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
12894 let registry = Arc::new(Registry::default());
12895 let control = ControlHandler::new(Arc::clone(®istry));
12896 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
12897 hello_via_sink(
12898 &control,
12899 &ctx,
12900 &mut rx,
12901 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
12902 )
12903 .await;
12904
12905 let captured = crate::router::test_log::captured_logs(&logs);
12906 let admissions: Vec<_> = captured
12907 .lines()
12908 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
12909 .collect();
12910 assert_eq!(
12911 admissions.len(),
12912 1,
12913 "legacy admission line must remain exactly once: {captured}"
12914 );
12915 assert!(admissions[0].contains("routable_provider=true"));
12916 assert!(admissions[0].contains("connection_id=777"));
12917 }
12918
12919 fn capability_manifest(
12920 module_id: &str,
12921 provides: &[&str],
12922 must_never_reach: &[&str],
12923 ) -> ModuleManifest {
12924 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
12925 manifest.capabilities = Some(CapabilityDeclarations {
12926 provides: provides
12927 .iter()
12928 .map(|capability| (*capability).to_string())
12929 .collect(),
12930 requires: Vec::new(),
12931 must_never_reach: must_never_reach
12932 .iter()
12933 .map(|capability| (*capability).to_string())
12934 .collect(),
12935 });
12936 manifest
12937 }
12938
12939 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
12940 Frame::build(
12941 FrameType::Hello,
12942 control_flags(),
12943 0,
12944 0,
12945 corr,
12946 serde_json::to_vec(&ModuleHelloBody {
12947 protocol_ver: manifest.protocol_ver,
12948 manifest,
12949 control_ops: None,
12950 launch_nonce: None,
12951 })
12952 .expect("capability test HELLO serializes"),
12953 )
12954 .expect("capability test HELLO frame builds")
12955 }
12956
12957 fn catalog_update_with_capabilities_frame(
12958 corr: u64,
12959 capabilities: CapabilityDeclarations,
12960 ) -> Frame {
12961 Frame::build(
12962 FrameType::Request,
12963 control_flags(),
12964 0,
12965 0,
12966 corr,
12967 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
12968 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
12969 capabilities: Some(capabilities),
12970 ready: None,
12971 })
12972 .expect("capability catalog.update serializes"),
12973 )
12974 .expect("capability catalog.update frame builds")
12975 }
12976
12977 async fn register_capability_manifest(
12978 handler: &ControlHandler,
12979 ctx: &RouteCtx,
12980 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
12981 manifest: ModuleManifest,
12982 corr: u64,
12983 ) {
12984 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
12985 }
12986
12987 async fn open_route_for_capability_test(
12988 handler: &ControlHandler,
12989 target_ctx: &RouteCtx,
12990 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
12991 client_connection_id: u64,
12992 corr: u64,
12993 target_module_id: &str,
12994 consumer_identity: Option<ConsumerIdentity>,
12995 ) -> (
12996 mpsc::Receiver<crate::router::OutboundFrame>,
12997 ModuleControlRequest,
12998 ) {
12999 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
13000 let route_handler = handler.clone();
13001 let target_module_id = target_module_id.to_string();
13002 let route_task = tokio::spawn(async move {
13003 route_handler
13004 .handle_control_frame(
13005 &client_ctx,
13006 route_open_frame_with_admission_facts(
13007 corr,
13008 &target_module_id,
13009 unique_project_root("admission-facts"),
13010 consumer_identity,
13011 None,
13012 ),
13013 )
13014 .await
13015 .expect("capability test route.open succeeds")
13016 });
13017 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
13018 .await
13019 .expect("capability test route.open must reach route.bind")
13020 .expect("target control receiver stays open");
13021 let bind_request: ModuleControlRequest =
13022 serde_json::from_slice(&bind.body).expect("route.bind decodes");
13023 handler
13024 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
13025 .await
13026 .expect("capability test route.bind ACK succeeds");
13027 assert!(route_task.await.expect("route.open task joins").is_empty());
13028 let opened = client_rx
13029 .recv()
13030 .await
13031 .expect("successful route.open publishes a response");
13032 assert!(matches!(
13033 serde_json::from_slice::<ClientControlResponse>(&opened.body),
13034 Ok(ClientControlResponse::RouteOpen { .. })
13035 ));
13036 (client_rx, bind_request)
13037 }
13038
13039 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
13040 assert_eq!(frame.header.ty, FrameType::Push);
13041 assert_eq!(frame.header.channel, 0);
13042 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
13043 .expect("route.closed control push decodes");
13044 let ClientControlPush::RouteClosed { channels, .. } = &push else {
13045 panic!("expected route.closed");
13046 };
13047 assert_eq!(channels.len(), 1, "exactly one violating route closed");
13048 let channels = channels.clone();
13049 assert_eq!(
13050 push,
13051 ClientControlPush::RouteClosed {
13052 module_id: target_module_id.to_string(),
13053 channels,
13054 reason: RouteCloseReason::CapabilityDenied,
13055 drained: false,
13056 abandoned: 0,
13057 excluded_subscriptions: 0,
13058 terminal: Some(false),
13059 }
13060 );
13061 }
13062
13063 #[tokio::test]
13064 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
13065 let registry = Arc::new(Registry::default());
13066 let forwarding = Arc::new(ForwardingTable::default());
13067 let supervisor = SupervisorHandle::new();
13068 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13069 let handler =
13070 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13071 .with_supervisor(supervisor);
13072 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
13073 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
13074 register_capability_manifest(
13075 &handler,
13076 &target_ctx,
13077 &mut target_rx,
13078 capability_manifest("target", &["credentials-provider/v1"], &[]),
13079 1,
13080 )
13081 .await;
13082 register_capability_manifest(
13083 &handler,
13084 &opener_ctx,
13085 &mut opener_rx,
13086 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13087 2,
13088 )
13089 .await;
13090
13091 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
13092 let replies = handler
13093 .handle_control_frame(
13094 &client_ctx,
13095 route_open_frame_with_admission_facts(
13096 3,
13097 "target",
13098 unique_project_root("admission-facts"),
13099 Some(ConsumerIdentity {
13100 module_id: "opener".to_string(),
13101 launch_nonce: "opener-nonce".to_string(),
13102 }),
13103 None,
13104 ),
13105 )
13106 .await
13107 .expect("denied route.open returns a typed frame");
13108 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13109 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13110 assert!(
13111 target_rx.try_recv().is_err(),
13112 "forbidden route.open must not relay route.bind"
13113 );
13114 }
13115
13116 #[tokio::test]
13117 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
13118 let registry = Arc::new(Registry::default());
13119 let forwarding = Arc::new(ForwardingTable::default());
13120 let supervisor = SupervisorHandle::new();
13121 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13122 let handler =
13123 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13124 .with_supervisor(supervisor);
13125 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
13126 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
13127 register_capability_manifest(
13128 &handler,
13129 &target_ctx,
13130 &mut target_rx,
13131 capability_manifest("target", &["credentials-provider/v1"], &[]),
13132 1,
13133 )
13134 .await;
13135 register_capability_manifest(
13136 &handler,
13137 &old_opener_ctx,
13138 &mut old_opener_rx,
13139 capability_manifest("opener", &[], &[]),
13140 2,
13141 )
13142 .await;
13143 let (mut client_rx, _) = open_route_for_capability_test(
13144 &handler,
13145 &target_ctx,
13146 &mut target_rx,
13147 712,
13148 3,
13149 "target",
13150 Some(ConsumerIdentity {
13151 module_id: "opener".to_string(),
13152 launch_nonce: "opener-nonce".to_string(),
13153 }),
13154 )
13155 .await;
13156 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13157
13158 handler
13159 .cleanup_connection(old_opener_ctx.connection_id)
13160 .expect("old opener registration cleans up");
13161 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
13162 register_capability_manifest(
13163 &handler,
13164 &new_opener_ctx,
13165 &mut new_opener_rx,
13166 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13167 4,
13168 )
13169 .await;
13170
13171 assert_capability_denied_push(
13172 client_rx
13173 .try_recv()
13174 .expect("HELLO deny addition must emit route.closed")
13175 .frame,
13176 "target",
13177 );
13178 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13179 assert!(matches!(
13180 target_rx.try_recv(),
13181 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13182 ));
13183 }
13184
13185 #[tokio::test]
13186 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
13187 let registry = Arc::new(Registry::default());
13188 let forwarding = Arc::new(ForwardingTable::default());
13189 let supervisor = SupervisorHandle::new();
13190 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13191 let handler =
13192 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13193 .with_supervisor(supervisor);
13194 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
13195 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
13196 register_capability_manifest(
13197 &handler,
13198 &target_ctx,
13199 &mut target_rx,
13200 capability_manifest("target", &[], &[]),
13201 1,
13202 )
13203 .await;
13204 register_capability_manifest(
13205 &handler,
13206 &opener_ctx,
13207 &mut opener_rx,
13208 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13209 2,
13210 )
13211 .await;
13212 let (mut client_rx, _) = open_route_for_capability_test(
13213 &handler,
13214 &target_ctx,
13215 &mut target_rx,
13216 722,
13217 3,
13218 "target",
13219 Some(ConsumerIdentity {
13220 module_id: "opener".to_string(),
13221 launch_nonce: "opener-nonce".to_string(),
13222 }),
13223 )
13224 .await;
13225 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13226
13227 let replies = handler
13228 .handle_control_frame(
13229 &target_ctx,
13230 catalog_update_with_capabilities_frame(
13231 4,
13232 CapabilityDeclarations {
13233 provides: vec!["credentials-provider/v1".to_string()],
13234 requires: Vec::new(),
13235 must_never_reach: Vec::new(),
13236 },
13237 ),
13238 )
13239 .await
13240 .expect("claim catalog.update succeeds");
13241 assert!(matches!(
13242 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
13243 Ok(ModuleControlResponseToModule::CatalogUpdate {})
13244 ));
13245 assert_capability_denied_push(
13246 client_rx
13247 .try_recv()
13248 .expect("claim addition must emit route.closed")
13249 .frame,
13250 "target",
13251 );
13252 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13253 assert!(matches!(
13254 target_rx.try_recv(),
13255 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13256 ));
13257 }
13258
13259 #[tokio::test]
13260 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
13261 let registry = Arc::new(Registry::default());
13262 let forwarding = Arc::new(ForwardingTable::default());
13263 let supervisor = SupervisorHandle::new();
13264 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13265 let handler =
13266 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13267 .with_supervisor(supervisor);
13268 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
13269 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
13270 register_capability_manifest(
13271 &handler,
13272 &target_ctx,
13273 &mut target_rx,
13274 capability_manifest("target", &["credentials-provider/v1"], &[]),
13275 1,
13276 )
13277 .await;
13278 register_capability_manifest(
13279 &handler,
13280 &opener_ctx,
13281 &mut opener_rx,
13282 capability_manifest("opener", &[], &[]),
13283 2,
13284 )
13285 .await;
13286 let (mut client_rx, _) = open_route_for_capability_test(
13287 &handler,
13288 &target_ctx,
13289 &mut target_rx,
13290 732,
13291 3,
13292 "target",
13293 Some(ConsumerIdentity {
13294 module_id: "opener".to_string(),
13295 launch_nonce: "opener-nonce".to_string(),
13296 }),
13297 )
13298 .await;
13299 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13300
13301 handler
13302 .handle_control_frame(
13303 &target_ctx,
13304 catalog_update_with_capabilities_frame(
13305 4,
13306 CapabilityDeclarations {
13307 provides: Vec::new(),
13308 requires: Vec::new(),
13309 must_never_reach: Vec::new(),
13310 },
13311 ),
13312 )
13313 .await
13314 .expect("claim removal catalog.update succeeds");
13315 assert_eq!(
13316 forwarding.active_binding_count().unwrap(),
13317 1,
13318 "removing an attested target claim must leave the route census unchanged"
13319 );
13320 assert!(
13321 client_rx.try_recv().is_err(),
13322 "claim removal must not emit route.closed capability_denied"
13323 );
13324 assert!(
13325 target_rx.try_recv().is_err(),
13326 "claim removal must not send the target a route GOODBYE"
13327 );
13328 }
13329
13330 #[tokio::test]
13333 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
13334 let registry = Arc::new(Registry::default());
13335 let forwarding = Arc::new(ForwardingTable::default());
13336 let supervisor = SupervisorHandle::new();
13337 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13338 let handler =
13339 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13340 .with_supervisor(supervisor);
13341 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
13342 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
13343 register_capability_manifest(
13344 &handler,
13345 &target_ctx,
13346 &mut target_rx,
13347 capability_manifest("target", &["credentials-provider/v1"], &[]),
13348 1,
13349 )
13350 .await;
13351 register_capability_manifest(
13352 &handler,
13353 &opener_ctx,
13354 &mut opener_rx,
13355 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13356 2,
13357 )
13358 .await;
13359
13360 let (_client_rx, bind) = open_route_for_capability_test(
13361 &handler,
13362 &target_ctx,
13363 &mut target_rx,
13364 742,
13365 3,
13366 "target",
13367 None,
13368 )
13369 .await;
13370 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
13371 panic!("direct scope-honesty route must bind");
13372 };
13373 assert_eq!(principal, Some(Principal::Direct));
13374 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13375 }
13376
13377 #[tokio::test]
13380 async fn must_never_reach_self_route_is_capability_forbidden() {
13381 let registry = Arc::new(Registry::default());
13382 let forwarding = Arc::new(ForwardingTable::default());
13383 let supervisor = SupervisorHandle::new();
13384 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
13385 let handler =
13386 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13387 .with_supervisor(supervisor);
13388 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
13389 register_capability_manifest(
13390 &handler,
13391 &self_ctx,
13392 &mut self_rx,
13393 capability_manifest(
13394 "self-provider",
13395 &["credentials-provider/v1"],
13396 &["credentials-provider/v1"],
13397 ),
13398 1,
13399 )
13400 .await;
13401
13402 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
13403 let replies = handler
13404 .handle_control_frame(
13405 &client_ctx,
13406 route_open_frame_with_admission_facts(
13407 2,
13408 "self-provider",
13409 unique_project_root("admission-facts"),
13410 Some(ConsumerIdentity {
13411 module_id: "self-provider".to_string(),
13412 launch_nonce: "self-nonce".to_string(),
13413 }),
13414 None,
13415 ),
13416 )
13417 .await
13418 .expect("self-route refusal returns a typed frame");
13419 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13420 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13421 assert!(
13422 self_rx.try_recv().is_err(),
13423 "self denial must not relay route.bind"
13424 );
13425 }
13426
13427 #[test]
13428 fn unsupported_channel_zero_frame_returns_error() {
13429 let handler = ControlHandler::default();
13430 let request = Frame::build(
13431 FrameType::Request,
13432 control_flags(),
13433 0,
13434 0,
13435 21,
13436 b"opaque".to_vec(),
13437 )
13438 .unwrap();
13439
13440 let response = handler
13441 .handle_control(ConnectionId::new(1), request)
13442 .unwrap();
13443
13444 assert_eq!(response[0].header.ty, FrameType::Error);
13445 assert_eq!(
13446 parse_error(&response[0])["code"],
13447 "unsupported_control_frame"
13448 );
13449 }
13450
13451 mod swap {
13456 use super::*;
13457
13458 const INCUMBENT: ConnectionId = ConnectionId::new(30);
13459 const CANDIDATE: ConnectionId = ConnectionId::new(40);
13460
13461 struct Swap {
13462 registry: Arc<Registry>,
13463 forwarding: Arc<ForwardingTable>,
13464 handler: ControlHandler,
13465 incumbent_ctx: RouteCtx,
13466 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13467 candidate_ctx: RouteCtx,
13468 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13469 }
13470
13471 async fn swap_with_incumbent() -> Swap {
13472 let registry = Arc::new(Registry::default());
13473 let forwarding = Arc::new(ForwardingTable::default());
13474 let handler =
13475 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13476 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
13477 hello_via_sink(
13478 &handler,
13479 &incumbent_ctx,
13480 &mut incumbent_rx,
13481 hello_frame("aft", PROTOCOL_VERSION, 7),
13482 )
13483 .await;
13484 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
13485 Swap {
13486 registry,
13487 forwarding,
13488 handler,
13489 incumbent_ctx,
13490 incumbent_rx,
13491 candidate_ctx,
13492 candidate_rx,
13493 }
13494 }
13495
13496 fn register_candidate(swap: &Swap, ready: Option<bool>) {
13497 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
13498 candidate_manifest.ready = ready;
13499 let registration = swap
13500 .registry
13501 .register_candidate_with_control_ops(
13502 candidate_manifest,
13503 PROTOCOL_VERSION,
13504 CANDIDATE,
13505 module_baseline_control_ops(),
13506 )
13507 .unwrap();
13508 swap.forwarding
13509 .register_candidate_module_connection(
13510 CANDIDATE,
13511 "aft".to_string(),
13512 PROTOCOL_VERSION,
13513 manifest_concurrency(®istration.manifest),
13514 swap.candidate_ctx.egress.clone(),
13515 )
13516 .unwrap();
13517 }
13518
13519 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
13520 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
13521 swap.registry.promote_candidate("aft").unwrap().unwrap();
13522 cutover.incumbent.unwrap()
13523 }
13524
13525 #[tokio::test]
13534 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
13535 let mut swap = swap_with_incumbent().await;
13536 let hold = Arc::new(std::sync::Barrier::new(2));
13537 let handler = swap
13538 .handler
13539 .clone()
13540 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
13541 handler
13542 .supervisor
13543 .open_swap("aft", "candidate-nonce".to_string());
13544 let hello = {
13545 let handler = handler.clone();
13546 let candidate_ctx = swap.candidate_ctx.clone();
13547 std::thread::spawn(move || {
13548 handler
13549 .handle_hello(
13550 candidate_ctx.connection_id,
13551 Some(candidate_ctx.egress.clone()),
13552 hello_frame_with_nonce(
13553 "aft",
13554 PROTOCOL_VERSION,
13555 8,
13556 Some("candidate-nonce"),
13557 ),
13558 )
13559 .unwrap()
13560 })
13561 };
13562
13563 hold.wait();
13564 assert!(swap
13567 .registry
13568 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
13569 .unwrap()
13570 .is_some());
13571 assert!(swap
13572 .registry
13573 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
13574 .unwrap()
13575 .is_none());
13576 assert!(swap
13577 .forwarding
13578 .module_endpoint_for_connection(CANDIDATE)
13579 .unwrap()
13580 .is_none());
13581
13582 hold.wait();
13583 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
13584 assert_eq!(
13585 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
13586 FrameType::HelloAck
13587 );
13588 let published = swap
13589 .registry
13590 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
13591 .unwrap()
13592 .expect("candidate is published once its HELLO completes");
13593 assert_eq!(published.connection_id, CANDIDATE);
13594 assert!(swap
13595 .forwarding
13596 .module_endpoint_for_connection(CANDIDATE)
13597 .unwrap()
13598 .is_some());
13599 }
13600
13601 fn keyed_total(counters: &Value, key: &str) -> u64 {
13602 counters[key]
13603 .as_object()
13604 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
13605 .unwrap_or(0)
13606 }
13607
13608 #[tokio::test]
13609 async fn replacement_logs_old_end_and_new_admission_once() {
13610 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13611 let mut swap = swap_with_incumbent().await;
13612 swap.handler
13613 .supervisor
13614 .open_swap("aft", "candidate-nonce".to_string());
13615 hello_via_sink(
13616 &swap.handler,
13617 &swap.candidate_ctx,
13618 &mut swap.candidate_rx,
13619 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
13620 )
13621 .await;
13622 cutover(&swap);
13623 swap.handler.cleanup_connection(INCUMBENT).unwrap();
13624
13625 let captured = crate::router::test_log::captured_logs(&logs);
13626 println!("captured replacement registry logs:\n{captured}");
13627 let new_admissions: Vec<_> = captured
13628 .lines()
13629 .filter(|line| {
13630 line.contains("connection_id=40")
13631 && line.contains("swap candidate registered; not routable until cutover")
13632 })
13633 .collect();
13634 assert_eq!(
13635 new_admissions.len(),
13636 1,
13637 "unexpected admission logs: {captured}"
13638 );
13639 assert!(new_admissions[0].contains("module_id=aft"));
13640 assert!(new_admissions[0].contains("connection_id=40"));
13641 assert!(
13642 new_admissions[0].contains("swap candidate registered; not routable until cutover")
13643 );
13644 assert!(new_admissions[0].contains("ready=true"));
13645
13646 let old_admissions: Vec<_> = captured
13647 .lines()
13648 .filter(|line| {
13649 line.contains("module registered module_id=aft ")
13650 && line.contains("connection_id=30")
13651 })
13652 .collect();
13653 assert_eq!(
13654 old_admissions.len(),
13655 1,
13656 "unexpected incumbent admission: {captured}"
13657 );
13658
13659 let old_ends: Vec<_> = captured
13660 .lines()
13661 .filter(|line| {
13662 line.contains("connection_id=30") && line.contains("module registration ended")
13663 })
13664 .collect();
13665 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
13666 assert!(old_ends[0].contains("module_id=aft"));
13667 assert!(old_ends[0].contains("connection_id=30"));
13668 assert!(old_ends[0].contains("reason=replaced"));
13669 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
13670 assert_eq!(
13671 captured
13672 .lines()
13673 .filter(|line| line.contains("module registration promoted"))
13674 .count(),
13675 1,
13676 "unexpected promotion logs: {captured}"
13677 );
13678 }
13679
13680 #[tokio::test]
13686 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
13687 let mut swap = swap_with_incumbent().await;
13688 let handler = swap.handler.clone();
13689
13690 let cotenant = ConnectionId::new(31);
13692 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
13693 let (cotenant_task, cotenant_bind) = relay_route_open(
13694 &handler,
13695 cotenant,
13696 &cotenant_ctx.egress,
13697 &mut swap.incumbent_rx,
13698 100,
13699 "aft",
13700 "swap-cotenant",
13701 )
13702 .await;
13703 handler
13704 .handle_control_frame(
13705 &swap.incumbent_ctx,
13706 route_bind_ack(cotenant_bind.header.corr),
13707 )
13708 .await
13709 .unwrap();
13710 assert!(cotenant_task.await.unwrap().is_empty());
13711 let (cotenant_channel, cotenant_epoch) =
13712 published_route(&cotenant_rx.recv().await.unwrap());
13713
13714 let caller = ConnectionId::new(32);
13716 let (caller_ctx, mut caller_rx) = route_ctx(caller);
13717 let (caller_task, caller_bind) = relay_route_open(
13718 &handler,
13719 caller,
13720 &caller_ctx.egress,
13721 &mut swap.incumbent_rx,
13722 101,
13723 "aft",
13724 "swap-caller",
13725 )
13726 .await;
13727 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
13728
13729 register_candidate(&swap, None);
13730 cutover(&swap);
13731
13732 let ack = handler
13734 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
13735 .await;
13736 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
13737 if module_loop_error.is_some() {
13738 handler.cleanup_connection(INCUMBENT).unwrap();
13741 }
13742
13743 assert!(
13745 cotenant_rx.try_recv().is_err(),
13746 "the co-tenant route on the incumbent was torn down by one late ack: \
13747 {module_loop_error:?}"
13748 );
13749 assert!(matches!(
13750 swap.forwarding
13751 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
13752 .unwrap(),
13753 DataRoute::Client(DataRouteState::Bound(_))
13754 ));
13755 assert_eq!(module_loop_error, None);
13756 assert!(swap
13757 .registry
13758 .get_module_by_connection(INCUMBENT)
13759 .unwrap()
13760 .is_some());
13761
13762 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
13764 .await
13765 .expect("the incumbent is told to drop the abandoned binding")
13766 .unwrap()
13767 .frame;
13768 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
13769 assert_eq!(goodbye.header.channel, abandoned_channel);
13770 assert_eq!(goodbye.header.epoch, abandoned_epoch);
13771 assert!(swap.incumbent_rx.try_recv().is_err());
13772
13773 let response = caller_task.await.unwrap();
13775 assert_eq!(response.len(), 1);
13776 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
13777 assert!(caller_rx.try_recv().is_err());
13778
13779 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
13783 let counters = handler.counters().snapshot();
13784 assert_eq!(
13785 keyed_total(&counters, "route_open_accepted_by_principal"),
13786 1
13787 );
13788 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
13789 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
13790 }
13791
13792 #[tokio::test]
13796 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
13797 let mut swap = swap_with_incumbent().await;
13798 register_candidate(&swap, None);
13799 let incumbent = cutover(&swap);
13800 swap.forwarding
13801 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
13802 .unwrap()
13803 .expect("the incumbent is still registered");
13804
13805 let client = ConnectionId::new(33);
13806 let (client_ctx, mut client_rx) = route_ctx(client);
13807 let route_handler = swap.handler.clone();
13808 let open_ctx = RouteCtx {
13809 connection_id: client,
13810 egress: client_ctx.egress.clone(),
13811 };
13812 let mut route_task = tokio::spawn(async move {
13813 route_handler
13814 .handle_control_frame(
13815 &open_ctx,
13816 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
13817 )
13818 .await
13819 .unwrap()
13820 });
13821 let bind = tokio::select! {
13822 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
13823 response = &mut route_task => {
13824 let response = response.unwrap();
13825 panic!(
13826 "post-cutover route.open was refused instead of relayed to the candidate: {}",
13827 parse_error(&response[0])["code"]
13828 );
13829 }
13830 };
13831 swap.handler
13832 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
13833 .await
13834 .unwrap();
13835 assert!(route_task.await.unwrap().is_empty());
13836 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
13837 match swap
13838 .forwarding
13839 .lookup_data_route(client, channel, epoch)
13840 .unwrap()
13841 {
13842 DataRoute::Client(DataRouteState::Bound(route)) => {
13843 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
13844 }
13845 other => panic!("expected a bound route on the candidate, got {other:?}"),
13846 }
13847 assert!(swap.incumbent_rx.try_recv().is_err());
13848 }
13849
13850 #[tokio::test]
13855 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
13856 let swap = swap_with_incumbent().await;
13857 register_candidate(&swap, Some(false));
13858 let update = Frame::build(
13859 FrameType::Request,
13860 control_flags(),
13861 0,
13862 0,
13863 55,
13864 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13865 provides: manifest("aft", PROTOCOL_VERSION).provides,
13866 capabilities: None,
13867 ready: Some(true),
13868 })
13869 .unwrap(),
13870 )
13871 .unwrap();
13872
13873 let replies = swap
13874 .handler
13875 .handle_control_frame(&swap.candidate_ctx, update)
13876 .await
13877 .unwrap();
13878
13879 assert_eq!(replies.len(), 1);
13880 assert_eq!(
13881 replies[0].header.ty,
13882 FrameType::Response,
13883 "candidate catalog.update was refused: {:?}",
13884 serde_json::from_slice::<Value>(&replies[0].body).ok()
13885 );
13886 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
13887 assert_eq!(
13888 swap.registry
13889 .get_module("aft")
13890 .unwrap()
13891 .unwrap()
13892 .connection_id,
13893 INCUMBENT
13894 );
13895 }
13896 }
13897
13898 mod swap_admission {
13902 use super::*;
13903
13904 const INCUMBENT_NONCE: &str = "incumbent-nonce";
13905 const CANDIDATE_NONCE: &str = "candidate-nonce";
13906
13907 fn handler_with_incumbent(
13908 module_id: &str,
13909 reserved: bool,
13910 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
13911 let registry = Arc::new(Registry::default());
13912 let supervisor = SupervisorHandle::new();
13913 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
13914 if reserved {
13915 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
13916 }
13917 let handler =
13918 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
13919 let incumbent = handler
13920 .handle_control(
13921 ConnectionId::new(1),
13922 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
13923 )
13924 .unwrap();
13925 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
13926 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
13927 (registry, supervisor, handler)
13928 }
13929
13930 #[test]
13936 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
13937 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
13938
13939 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
13940 let replies = handler
13941 .handle_control(
13942 ConnectionId::new(connection),
13943 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
13944 )
13945 .unwrap();
13946 assert_eq!(replies[0].header.ty, FrameType::Error);
13947 assert_eq!(
13948 parse_error(&replies[0])["code"],
13949 "swap_token_invalid",
13950 "nonce {nonce:?}"
13951 );
13952 }
13953 assert!(registry.get_candidate("aft").unwrap().is_none());
13954 assert_eq!(
13955 registry.get_module("aft").unwrap().unwrap().connection_id,
13956 ConnectionId::new(1)
13957 );
13958
13959 let admitted = handler
13962 .handle_control(
13963 ConnectionId::new(4),
13964 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
13965 )
13966 .unwrap();
13967 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
13968 assert_eq!(
13969 registry
13970 .get_candidate("aft")
13971 .unwrap()
13972 .unwrap()
13973 .connection_id,
13974 ConnectionId::new(4)
13975 );
13976 assert_eq!(
13977 registry.get_module("aft").unwrap().unwrap().connection_id,
13978 ConnectionId::new(1),
13979 "the candidate must not take the active slot"
13980 );
13981 let replayed = handler
13982 .handle_control(
13983 ConnectionId::new(5),
13984 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
13985 )
13986 .unwrap();
13987 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
13988
13989 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
13993 let squatter = handler
13994 .handle_control(
13995 ConnectionId::new(6),
13996 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
13997 )
13998 .unwrap();
13999 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
14000 assert!(
14001 registry.get_module("aft").unwrap().is_none(),
14002 "a squatter took the active slot of an id being swapped"
14003 );
14004 }
14005
14006 #[test]
14011 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
14012 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
14013
14014 let replies = handler
14015 .handle_control(
14016 ConnectionId::new(2),
14017 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14018 )
14019 .unwrap();
14020
14021 assert_eq!(
14022 replies[0].header.ty,
14023 FrameType::HelloAck,
14024 "reserved candidate refused: {:?}",
14025 serde_json::from_slice::<Value>(&replies[0].body).ok()
14026 );
14027 assert_eq!(
14028 registry
14029 .get_candidate("vault")
14030 .unwrap()
14031 .unwrap()
14032 .connection_id,
14033 ConnectionId::new(2)
14034 );
14035 }
14036
14037 #[test]
14040 fn without_an_open_swap_the_ordinary_gates_decide() {
14041 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
14042 supervisor.close_swap("vault");
14043
14044 let candidate = handler
14045 .handle_control(
14046 ConnectionId::new(2),
14047 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14048 )
14049 .unwrap();
14050 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
14051 let duplicate = handler
14052 .handle_control(
14053 ConnectionId::new(3),
14054 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
14055 )
14056 .unwrap();
14057 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14058 assert!(registry.get_candidate("vault").unwrap().is_none());
14059 }
14060 }
14061
14062 mod scopes {
14066 mod activity {
14067 include!("activity_scope_tests.rs");
14068 }
14069 mod delta {
14070 include!("scope_handler_delta_tests.rs");
14071 }
14072 mod call_modes {
14073 include!("scope_call_mode_tests.rs");
14074 }
14075
14076 use subc_protocol::scope::{
14077 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
14078 ScopeStatus,
14079 };
14080
14081 use super::*;
14082
14083 const OWNER: &str = "prefrontal-core";
14084
14085 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
14086 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
14087 }
14088
14089 async fn call(
14090 handler: &ControlHandler,
14091 ctx: &RouteCtx,
14092 request: &ModuleControlRequestFromModule,
14093 ) -> Frame {
14094 let body = serde_json::to_vec(request).unwrap();
14095 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
14096 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
14097 assert_eq!(replies.len(), 1, "{replies:?}");
14098 replies.pop().unwrap()
14099 }
14100
14101 async fn sync(
14102 handler: &ControlHandler,
14103 ctx: &RouteCtx,
14104 generation: u64,
14105 scopes: Vec<ScopeRecord>,
14106 ) -> Result<ModuleControlResponseToModule, String> {
14107 let reply = call(
14108 handler,
14109 ctx,
14110 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
14111 )
14112 .await;
14113 match reply.header.ty {
14114 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
14115 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
14116 }
14117 }
14118
14119 async fn describe(
14120 handler: &ControlHandler,
14121 ctx: &RouteCtx,
14122 owner: &str,
14123 scope_ref: &str,
14124 ) -> ModuleControlResponseToModule {
14125 let reply = call(
14126 handler,
14127 ctx,
14128 &ModuleControlRequestFromModule::ScopeDescribe {
14129 owner: Principal::Reserved {
14130 module_id: owner.to_string(),
14131 },
14132 scope_ref: scope_ref.to_string(),
14133 },
14134 )
14135 .await;
14136 assert_eq!(
14137 reply.header.ty,
14138 FrameType::Response,
14139 "{:?}",
14140 parse_error(&reply)
14141 );
14142 serde_json::from_slice(&reply.body).unwrap()
14143 }
14144
14145 async fn module(
14147 handler: &ControlHandler,
14148 connection: u64,
14149 module_id: &str,
14150 nonce: Option<&str>,
14151 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
14152 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
14153 hello_via_sink(
14154 handler,
14155 &ctx,
14156 &mut rx,
14157 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
14158 )
14159 .await;
14160 (ctx, rx)
14161 }
14162
14163 #[tokio::test]
14166 async fn a_client_connection_cannot_sync_or_describe() {
14167 let handler = ControlHandler::new(Arc::new(Registry::default()));
14168 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
14169 for request in [
14170 ModuleControlRequestFromModule::ScopeSync {
14171 generation: 1,
14172 scopes: vec![head("s", 1)],
14173 },
14174 ModuleControlRequestFromModule::ScopeDescribe {
14175 owner: Principal::Direct,
14176 scope_ref: "s".to_string(),
14177 },
14178 ] {
14179 let reply = call(&handler, &ctx, &request).await;
14180 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
14181 }
14182 assert!(
14183 !handler
14184 .scopes
14185 .read()
14186 .unwrap()
14187 .describe(
14188 &Principal::Reserved {
14189 module_id: OWNER.to_string()
14190 },
14191 "s"
14192 )
14193 .owner_synced
14194 );
14195 }
14196
14197 #[tokio::test]
14200 async fn a_module_without_a_supervised_launch_cannot_sync() {
14201 let handler = ControlHandler::new(Arc::new(Registry::default()));
14202 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
14203 assert_eq!(
14204 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
14205 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14206 );
14207 }
14208
14209 #[tokio::test]
14210 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
14211 let supervisor = SupervisorHandle::new();
14212 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14213 let handler = ControlHandler::new(Arc::new(Registry::default()))
14214 .with_supervisor(supervisor.clone());
14215 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14216 sync(&handler, &incumbent, 1, vec![head("s", 1)])
14217 .await
14218 .expect("the current launch syncs");
14219
14220 supervisor.open_swap(OWNER, "n2".to_string());
14223 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
14224 assert_eq!(
14225 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14226 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14227 );
14228 sync(&handler, &incumbent, 2, vec![head("s", 1)])
14229 .await
14230 .expect("the serving owner syncs during the swap");
14231
14232 supervisor.close_swap(OWNER);
14235 assert_eq!(
14236 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14237 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14238 );
14239 sync(&handler, &incumbent, 3, vec![head("s", 1)])
14240 .await
14241 .expect("the serving owner syncs after the rollback");
14242 handler.cleanup_connection(candidate.connection_id).unwrap();
14243
14244 supervisor.open_swap(OWNER, "n3".to_string());
14249 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
14250 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
14251 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
14252 .await
14253 .expect("the promoted launch takes authority");
14254 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
14255 panic!("unexpected reply {reply:?}");
14256 };
14257 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
14258 assert_eq!(
14259 sync(&handler, &incumbent, 4, Vec::new()).await,
14260 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14261 );
14262 }
14263
14264 #[tokio::test]
14267 async fn closing_the_authority_connection_frees_sync_authority() {
14268 let supervisor = SupervisorHandle::new();
14269 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14270 let handler = ControlHandler::new(Arc::new(Registry::default()))
14271 .with_supervisor(supervisor.clone());
14272 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14273 sync(&handler, &first, 10, vec![head("s", 1)])
14274 .await
14275 .unwrap();
14276 handler.cleanup_connection(first.connection_id).unwrap();
14277
14278 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
14279 sync(&handler, &second, 1, vec![head("s", 1)])
14280 .await
14281 .expect("the next connection takes the released authority");
14282 }
14283
14284 #[tokio::test]
14285 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
14286 let supervisor = SupervisorHandle::new();
14287 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14288 let handler =
14289 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
14290 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
14291 sync(&handler, &first, 10, vec![head("s", 1)])
14292 .await
14293 .unwrap();
14294 handler
14295 .handle_control_frame(
14296 &first,
14297 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
14298 )
14299 .await
14300 .unwrap();
14301 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
14302 sync(&handler, &second, 1, vec![head("s", 1)])
14303 .await
14304 .expect("GOODBYE releases authority even if the old socket remains open");
14305 }
14306
14307 #[tokio::test]
14308 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
14309 let registry = Arc::new(Registry::default());
14310 let supervisor_handle = SupervisorHandle::new();
14311 let supervisor =
14312 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14313 .with_handle(supervisor_handle.clone())
14314 .with_daemon_incarnation("incarnation-7".to_string());
14315 supervisor
14318 .supervise_configured(
14319 ModuleSpec {
14320 module_id: OWNER.to_string(),
14321 program: PathBuf::from("/nonexistent/prefrontal-core"),
14322 args: Vec::new(),
14323 env: Vec::new(),
14324 reserved: false,
14325 reserved_prefixes: Vec::new(),
14326 protocol: ModuleProtocol::Subc,
14327 overlap: Default::default(),
14328 },
14329 false,
14330 )
14331 .unwrap();
14332 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
14333 let handler =
14334 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14335 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
14336
14337 let ModuleControlResponseToModule::ScopeDescribe {
14339 status,
14340 daemon_incarnation,
14341 owner_synced,
14342 owner_configured,
14343 scope,
14344 ..
14345 } = describe(&handler, &reader, OWNER, "s").await
14346 else {
14347 panic!("not a describe reply");
14348 };
14349 assert_eq!(status, ScopeStatus::NotLive);
14350 assert_eq!(daemon_incarnation, "incarnation-7");
14351 assert!(!owner_synced);
14352 assert!(owner_configured);
14353 assert!(scope.is_none());
14354
14355 let ModuleControlResponseToModule::ScopeDescribe {
14358 status,
14359 owner_configured,
14360 ..
14361 } = describe(&handler, &reader, "ghost", "s").await
14362 else {
14363 panic!("not a describe reply");
14364 };
14365 assert_eq!(status, ScopeStatus::NotLive);
14366 assert!(!owner_configured);
14367
14368 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
14370 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
14371 let ModuleControlResponseToModule::ScopeDescribe {
14372 status,
14373 scope_epoch,
14374 owner_synced,
14375 scope,
14376 ..
14377 } = describe(&handler, &reader, OWNER, "s").await
14378 else {
14379 panic!("not a describe reply");
14380 };
14381 assert_eq!(status, ScopeStatus::Live);
14382 assert_eq!(scope_epoch, Some(4));
14383 assert!(owner_synced);
14384 let stamp = scope.expect("a live scope carries its stamp");
14385 assert!(
14386 stamp.owner_authorized,
14387 "prefrontal-core is the default authority"
14388 );
14389 assert_eq!(stamp.kind, ScopeKind::Head);
14390 }
14391
14392 #[tokio::test]
14393 async fn scope_authority_owners_decides_owner_authorized() {
14394 let supervisor = SupervisorHandle::new();
14395 supervisor.set_spawn_nonce("broca", "b1".to_string());
14396 let handler = ControlHandler::new(Arc::new(Registry::default()))
14397 .with_supervisor(supervisor)
14398 .with_scope_authority_owners(vec!["broca".to_string()]);
14399 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
14400 let mut gated = head("s", 1);
14401 gated.attributes.agent_id = Some("agent".to_string());
14402 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
14403 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
14404 describe(&handler, &broca, "broca", "s").await
14405 else {
14406 panic!("not a describe reply");
14407 };
14408 assert!(scope.unwrap().owner_authorized);
14409 }
14410
14411 #[tokio::test]
14415 async fn scope_ops_and_the_scopes_capability_are_advertised() {
14416 let handler = ControlHandler::new(Arc::new(Registry::default()));
14417 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
14418 let ack = hello_via_sink(
14419 &handler,
14420 &ctx,
14421 &mut rx,
14422 hello_frame("m", PROTOCOL_VERSION, 1),
14423 )
14424 .await;
14425 let ack = parse_ack(&ack);
14426 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
14427 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
14428 }
14429 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
14430
14431 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
14432 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
14433 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
14434 let reply = handler
14435 .handle_control_frame(&client, frame)
14436 .await
14437 .unwrap()
14438 .pop()
14439 .unwrap();
14440 let ClientControlResponse::ServerDescribe { capabilities, .. } =
14441 serde_json::from_slice(&reply.body).unwrap()
14442 else {
14443 panic!("not a server.describe reply");
14444 };
14445 assert!(
14446 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
14447 "{capabilities:?}"
14448 );
14449 }
14450
14451 const PLEXUS: &str = "plexus";
14454 const OTHER: &str = "other";
14455 const AFT: &str = "aft";
14456 const BROCA: &str = "broca";
14457 const MAGIC: &str = "magic-context";
14458
14459 fn nonce(module_id: &str) -> String {
14460 format!("nonce-{module_id}")
14461 }
14462
14463 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
14464 let (tx, rx) = mpsc::channel(64);
14465 (
14466 RouteCtx {
14467 connection_id: ConnectionId::new(connection),
14468 egress: FrameSink::new(tx),
14469 },
14470 rx,
14471 )
14472 }
14473
14474 struct Rig {
14478 handler: ControlHandler,
14479 forwarding: Arc<ForwardingTable>,
14480 owner: RouteCtx,
14481 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14482 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
14483 generation: u64,
14484 next_connection: u64,
14485 _supervisor: Supervisor,
14486 }
14487
14488 async fn rig() -> Rig {
14489 rig_with_flow_support(true).await
14490 }
14491
14492 async fn rig_with_flow_support(flow_support: bool) -> Rig {
14493 rig_with_scope_capabilities(flow_support, false).await
14494 }
14495
14496 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
14497 let mut provides = Vec::new();
14498 if flow_support {
14499 provides.push("flow-scopes/v1");
14500 }
14501 if run_support {
14502 provides.push("agent-run-scopes/v1");
14503 }
14504 rig_with_target_capabilities(&provides).await
14505 }
14506
14507 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
14508 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
14509 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
14510 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
14513 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
14514 Frame::build(
14515 FrameType::Hello,
14516 control_flags(),
14517 0,
14518 0,
14519 connection,
14520 serde_json::to_vec(&body).unwrap(),
14521 )
14522 .unwrap()
14523 }
14524
14525 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
14526 let registry = Arc::new(Registry::default());
14527 let forwarding = Arc::new(ForwardingTable::default());
14528 let supervisor_handle = SupervisorHandle::new();
14529 let supervisor =
14530 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14531 .with_handle(supervisor_handle.clone());
14532 supervisor
14533 .supervise_configured(
14534 ModuleSpec {
14535 module_id: OWNER.to_string(),
14536 program: PathBuf::from("/nonexistent/prefrontal-core"),
14537 args: Vec::new(),
14538 env: Vec::new(),
14539 reserved: false,
14540 reserved_prefixes: Vec::new(),
14541 protocol: ModuleProtocol::Subc,
14542 overlap: Default::default(),
14543 },
14544 false,
14545 )
14546 .unwrap();
14547 for module_id in [OWNER, AFT, BROCA, MAGIC] {
14548 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
14549 }
14550 let handler =
14551 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14552 .with_supervisor(supervisor_handle);
14553 let (owner, mut owner_rx) = wide_ctx(1);
14554 hello_via_sink(
14555 &handler,
14556 &owner,
14557 &mut owner_rx,
14558 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
14559 )
14560 .await;
14561 let mut modules = BTreeMap::new();
14562 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
14563 let (ctx, mut rx) = wide_ctx(connection);
14564 let hello = hello_with_capabilities(module_id, connection, provides);
14565 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
14566 modules.insert(module_id.to_string(), (ctx, rx));
14567 }
14568 Rig {
14569 handler,
14570 forwarding,
14571 owner,
14572 _owner_rx: owner_rx,
14573 modules,
14574 generation: 0,
14575 next_connection: 100,
14576 _supervisor: supervisor,
14577 }
14578 }
14579
14580 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
14581 ScopeCarrier::new(Principal::Reserved {
14582 module_id: module_id.to_string(),
14583 })
14584 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
14585 }
14586
14587 fn session(scope_epoch: u64) -> ScopeRecord {
14590 let mut record = head("s", scope_epoch);
14591 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
14592 record.attributes.agent_id = Some("agent-1".to_string());
14593 record.attributes.delegates = true;
14594 record
14595 }
14596
14597 impl Rig {
14598 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
14599 self.generation += 1;
14600 sync(&self.handler, &self.owner, self.generation, scopes)
14601 .await
14602 .expect("the owner's sync is accepted");
14603 }
14604
14605 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
14606 ScopeSelector {
14607 owner: Principal::Reserved {
14608 module_id: OWNER.to_string(),
14609 },
14610 scope_ref: scope_ref.to_string(),
14611 scope_epoch,
14612 }
14613 }
14614
14615 fn open_frame(
14616 &mut self,
14617 opener: Option<&str>,
14618 target: &str,
14619 scope: Option<ScopeSelector>,
14620 ) -> (
14621 RouteCtx,
14622 mpsc::Receiver<crate::router::OutboundFrame>,
14623 Frame,
14624 ) {
14625 self.next_connection += 1;
14626 let (ctx, rx) = wide_ctx(self.next_connection);
14627 let root = unique_project_root("scoped-open");
14628 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
14629 target: RouteTarget::ToolProvider {
14630 module_id: target.to_string(),
14631 },
14632 identity: BindIdentity::new(
14633 root.path().to_path_buf(),
14634 "unit".to_string(),
14635 "session".to_string(),
14636 ),
14637 consumer_identity: opener.map(|module_id| ConsumerIdentity {
14638 module_id: module_id.to_string(),
14639 launch_nonce: nonce(module_id),
14640 }),
14641 consumer_capabilities: None,
14642 role_versions: None,
14643 admission_facts: None,
14644 scope,
14645 })
14646 .unwrap();
14647 let frame = Frame::build(
14648 FrameType::Request,
14649 control_flags(),
14650 0,
14651 0,
14652 self.next_connection,
14653 body,
14654 )
14655 .unwrap();
14656 (ctx, rx, frame)
14657 }
14658
14659 async fn refused(
14661 &mut self,
14662 opener: Option<&str>,
14663 target: &str,
14664 scope: Option<ScopeSelector>,
14665 ) -> String {
14666 self.refusal_body(opener, target, scope).await["code"]
14667 .as_str()
14668 .unwrap()
14669 .to_string()
14670 }
14671
14672 async fn refusal_body(
14673 &mut self,
14674 opener: Option<&str>,
14675 target: &str,
14676 scope: Option<ScopeSelector>,
14677 ) -> Value {
14678 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
14679 let replies = tokio::time::timeout(
14680 Duration::from_secs(2),
14681 self.handler.handle_control_frame(&ctx, frame),
14682 )
14683 .await
14684 .expect("the open must be refused before waiting for a bind ack")
14685 .unwrap();
14686 assert_eq!(replies.len(), 1, "{replies:?}");
14687 assert_eq!(replies[0].header.ty, FrameType::Error);
14688 let (_, module_rx) = self.modules.get_mut(target).unwrap();
14689 assert!(
14690 module_rx.try_recv().is_err(),
14691 "a refused open relays nothing"
14692 );
14693 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
14694 parse_error(&replies[0])
14695 }
14696
14697 async fn relayed(
14699 &mut self,
14700 opener: Option<&str>,
14701 target: &str,
14702 scope: Option<ScopeSelector>,
14703 ) -> Relayed {
14704 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
14705 let handler = self.handler.clone();
14706 let task_ctx = ctx.clone();
14707 let task = tokio::spawn(async move {
14708 handler
14709 .handle_control_frame(&task_ctx, frame)
14710 .await
14711 .unwrap()
14712 });
14713 let (_, module_rx) = self.modules.get_mut(target).unwrap();
14714 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
14715 .await
14716 .expect("the target receives the relayed route.bind")
14717 .unwrap()
14718 .frame;
14719 Relayed {
14720 target: target.to_string(),
14721 client: ctx,
14722 client_rx: rx,
14723 task,
14724 bind,
14725 }
14726 }
14727
14728 async fn ack(&self, relayed: &Relayed) {
14729 let (module, _) = &self.modules[&relayed.target];
14730 self.handler
14731 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
14732 .await
14733 .unwrap();
14734 }
14735
14736 async fn bound(
14738 &mut self,
14739 opener: Option<&str>,
14740 target: &str,
14741 scope: Option<ScopeSelector>,
14742 ) -> Bound {
14743 let relayed = self.relayed(opener, target, scope).await;
14744 self.ack(&relayed).await;
14745 let Relayed {
14746 target,
14747 client,
14748 mut client_rx,
14749 task,
14750 bind,
14751 } = relayed;
14752 assert!(
14753 task.await.unwrap().is_empty(),
14754 "the open is answered by commit"
14755 );
14756 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
14757 Bound {
14758 target,
14759 client,
14760 client_rx,
14761 channel,
14762 epoch,
14763 bind,
14764 }
14765 }
14766
14767 fn live(&self, route: &Bound) -> bool {
14768 matches!(
14769 self.forwarding
14770 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
14771 .unwrap(),
14772 DataRoute::Client(DataRouteState::Bound(_))
14773 )
14774 }
14775 }
14776
14777 struct Relayed {
14778 target: String,
14779 client: RouteCtx,
14780 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14781 task: tokio::task::JoinHandle<Vec<Frame>>,
14782 bind: Frame,
14783 }
14784
14785 struct Bound {
14786 target: String,
14787 client: RouteCtx,
14788 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14789 channel: u16,
14790 epoch: u32,
14791 bind: Frame,
14792 }
14793
14794 impl Bound {
14795 fn closed_reason(&mut self) -> RouteCloseReason {
14798 let mut reason = None;
14799 let mut goodbye = false;
14800 while let Ok(outbound) = self.client_rx.try_recv() {
14801 let frame = outbound.frame;
14802 match frame.header.ty {
14803 FrameType::Goodbye => {
14804 assert_eq!(
14805 (frame.header.channel, frame.header.epoch),
14806 (self.channel, self.epoch)
14807 );
14808 goodbye = true;
14809 }
14810 FrameType::Push => {
14811 let ClientControlPush::RouteClosed {
14812 reason: r,
14813 module_id,
14814 ..
14815 } = serde_json::from_slice(&frame.body).unwrap()
14816 else {
14817 panic!("unexpected push");
14818 };
14819 assert_eq!(module_id, self.target);
14820 reason = Some(r);
14821 }
14822 other => panic!("unexpected frame {other:?}"),
14823 }
14824 }
14825 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
14826 reason.expect("the client is told why the route closed")
14827 }
14828
14829 fn untouched(&mut self) -> bool {
14830 self.client_rx.try_recv().is_err()
14831 }
14832
14833 fn stamp(&self) -> Option<ScopeStamp> {
14834 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
14835 ModuleControlRequest::RouteBind { scope, .. } => scope,
14836 other => panic!("expected a route.bind, got {other:?}"),
14837 }
14838 }
14839 }
14840
14841 #[tokio::test]
14842 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
14843 ) {
14844 let mut rig = rig().await;
14845 let mut record = session(1);
14846 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
14847 record.child_owners = vec![Principal::Reserved {
14848 module_id: MAGIC.to_string(),
14849 }];
14850 rig.sync(vec![record]).await;
14851 let scope = || Some(rig_selector("s", Some(1)));
14852
14853 rig.bound(Some(OWNER), PLEXUS, scope()).await;
14856 rig.bound(Some(AFT), OTHER, scope()).await;
14857 rig.bound(Some(BROCA), PLEXUS, scope()).await;
14858
14859 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
14863 assert_eq!(
14864 rig.refused(opener, target, scope()).await,
14865 error_codes::SCOPE_NOT_CARRIER,
14866 "{opener:?} -> {target}"
14867 );
14868 }
14869 }
14870
14871 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
14872 ScopeSelector {
14873 owner: Principal::Reserved {
14874 module_id: OWNER.to_string(),
14875 },
14876 scope_ref: scope_ref.to_string(),
14877 scope_epoch,
14878 }
14879 }
14880
14881 #[tokio::test]
14882 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
14883 let mut rig = rig().await;
14884 rig.sync(vec![session(1)]).await;
14885 for opener in [OWNER, AFT] {
14886 assert_eq!(
14887 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
14888 .await,
14889 error_codes::SCOPE_EPOCH_REQUIRED,
14890 "{opener}"
14891 );
14892 }
14893 }
14894
14895 #[tokio::test]
14896 async fn admission_separates_not_synced_not_live_and_ended() {
14897 let mut rig = rig().await;
14898 let code = rig
14900 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14901 .await;
14902 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
14903 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
14904
14905 let ghost = ScopeSelector {
14907 owner: Principal::Reserved {
14908 module_id: "ghost".to_string(),
14909 },
14910 scope_ref: "s".to_string(),
14911 scope_epoch: Some(1),
14912 };
14913 assert_eq!(
14914 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
14915 error_codes::SCOPE_NOT_LIVE
14916 );
14917
14918 rig.sync(vec![session(2)]).await;
14919 assert_eq!(
14920 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
14921 .await,
14922 error_codes::SCOPE_NOT_LIVE
14923 );
14924 for epoch in [1, 3] {
14925 assert_eq!(
14926 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
14927 .await,
14928 error_codes::SCOPE_ENDED,
14929 "epoch {epoch}"
14930 );
14931 }
14932 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
14934 .await;
14935 }
14936
14937 #[tokio::test]
14938 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
14939 let mut rig = rig().await;
14940 rig.sync(vec![session(1)]).await;
14941 let route = rig
14942 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14943 .await;
14944 let stamp = route.stamp().expect("a scoped bind carries the stamp");
14945 assert_eq!(stamp.scope_ref, "s");
14946 assert_eq!(stamp.scope_epoch, 1);
14947 assert_eq!(stamp.kind, ScopeKind::Head);
14948 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
14949 assert!(stamp.attributes.delegates);
14950 assert!(stamp.owner_authorized);
14951 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
14952 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
14953
14954 let (broca, mut broca_rx) = wide_ctx(50);
14957 hello_via_sink(
14958 &rig.handler,
14959 &broca,
14960 &mut broca_rx,
14961 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
14962 )
14963 .await;
14964 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
14965 .await
14966 .unwrap();
14967 let own = ScopeSelector {
14968 owner: Principal::Reserved {
14969 module_id: BROCA.to_string(),
14970 },
14971 scope_ref: "b".to_string(),
14972 scope_epoch: Some(1),
14973 };
14974 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
14975 assert!(!route.stamp().unwrap().owner_authorized);
14976 }
14977
14978 #[tokio::test]
14979 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
14980 let mut rig = rig().await;
14981 let mut record = head("s", 1);
14982 record.carriers = vec![carrier(AFT, None)];
14983 let flow_id = "Flow:run-7/step_2!~";
14984 record.attributes.flow_id = Some(flow_id.to_string());
14985 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
14986 .await
14987 .unwrap();
14988 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
14989 panic!("not a sync reply");
14990 };
14991 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
14992 let route = rig
14993 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
14994 .await;
14995 assert!(rig.live(&route), "the stamped bind committed");
14996 let stamp = route.stamp().expect("a flow scope carries a stamp");
14997 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
14998 assert_eq!(stamp.attributes.agent_id, None);
14999 assert!(!stamp.attributes.delegates);
15000 assert!(stamp.owner_authorized);
15001 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15002 assert_eq!(unscoped.stamp(), None);
15003 }
15004
15005 #[tokio::test]
15006 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
15007 let mut rig = rig_with_flow_support(false).await;
15008 let mut record = session(1);
15009 record.attributes.flow_id = Some("flow:7".to_string());
15010 rig.sync(vec![record]).await;
15011 for opener in [OWNER, AFT] {
15012 let body = rig
15013 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15014 .await;
15015 assert_eq!(body["code"], "target_flow_unsupported");
15016 let message = body["message"].as_str().unwrap();
15017 for required in [PLEXUS, "flow-scopes/v1"] {
15018 assert!(message.contains(required), "{message}");
15019 }
15020 }
15021 }
15022
15023 #[tokio::test]
15024 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
15025 let mut rig = rig_with_flow_support(true).await;
15026 let mut record = session(1);
15027 record.attributes.flow_id = Some("flow:7".to_string());
15028 rig.sync(vec![record]).await;
15029 for opener in [OWNER, AFT] {
15030 let route = rig
15031 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15032 .await;
15033 assert!(rig.live(&route));
15034 assert_eq!(
15035 route.stamp().unwrap().attributes.flow_id.as_deref(),
15036 Some("flow:7")
15037 );
15038 }
15039 }
15040
15041 #[tokio::test]
15042 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
15043 use std::future::Future;
15044
15045 let mut rig = rig_with_flow_support(true).await;
15046 let mut record = session(1);
15047 record.attributes.flow_id = Some("flow:7".to_string());
15048 rig.sync(vec![record]).await;
15049 let (client, mut client_rx, frame) =
15050 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
15051 for _ in 0..64 {
15054 client.egress.try_send(route_bind_ack(1)).unwrap();
15055 }
15056 let handler = rig.handler.clone();
15057 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15058 std::future::poll_fn(|cx| {
15059 assert!(open.as_mut().poll(cx).is_pending());
15060 std::task::Poll::Ready(())
15061 })
15062 .await;
15063 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15064
15065 let old_connection = rig.modules[PLEXUS].0.connection_id;
15066 rig.handler.cleanup_connection(old_connection).unwrap();
15067 let (replacement, mut replacement_rx) = wide_ctx(200);
15068 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
15069 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15070 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15071 let hello = Frame::build(
15072 FrameType::Hello,
15073 control_flags(),
15074 0,
15075 0,
15076 200,
15077 serde_json::to_vec(&body).unwrap(),
15078 )
15079 .unwrap();
15080 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
15081
15082 client_rx.try_recv().unwrap();
15083 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15084 .await
15085 .expect("the replacement is refused without waiting for a bind ack")
15086 .unwrap();
15087 assert_eq!(replies.len(), 1);
15088 let body = parse_error(&replies[0]);
15089 assert_eq!(body["code"], "target_flow_unsupported");
15090 for required in [PLEXUS, "flow-scopes/v1"] {
15091 assert!(body["message"].as_str().unwrap().contains(required));
15092 }
15093 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
15094 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15095 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15096 assert!(rig
15097 .handler
15098 .registry
15099 .get_module_by_connection(replacement.connection_id)
15100 .unwrap()
15101 .is_some());
15102 }
15103
15104 #[tokio::test]
15105 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
15106 {
15107 let mut rig = rig_with_flow_support(false).await;
15108 rig.sync(vec![session(1)]).await;
15109 let route = rig
15110 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15111 .await;
15112 assert!(rig.live(&route));
15113 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
15114 let mut record = session(1);
15115 record.attributes.flow_id = Some("flow:7".to_string());
15116 rig.sync(vec![record]).await;
15117 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
15118 assert!(rig.live(&unscoped));
15119 assert_eq!(unscoped.stamp(), None);
15120 }
15121
15122 #[tokio::test]
15123 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
15124 let mut rig = rig().await;
15125 let mut record = session(1);
15126 record.attributes.flow_id = Some("flow:7".to_string());
15127 rig.sync(vec![record.clone()]).await;
15128 let before = rig.forwarding.published_scope_tag(OWNER, "s");
15129 let mut owner_route = rig
15130 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15131 .await;
15132 let mut carrier_route = rig
15133 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
15134 .await;
15135 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15136 rig.sync(vec![record.clone()]).await;
15137 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
15138 assert!(rig.live(&owner_route) && owner_route.untouched());
15139 assert!(rig.live(&carrier_route) && carrier_route.untouched());
15140
15141 record.attributes.flow_id = Some("flow:8".to_string());
15142 rig.sync(vec![record]).await;
15143 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
15144 let before = before.unwrap();
15145 assert_eq!(after.scope_epoch, before.scope_epoch);
15146 assert!(after.version > before.version);
15147 for route in [&mut owner_route, &mut carrier_route] {
15148 assert!(!rig.live(route));
15149 assert_eq!(
15150 route.closed_reason(),
15151 RouteCloseReason::ScopeDelegationChanged
15152 );
15153 }
15154 assert!(rig.live(&unscoped) && unscoped.untouched());
15155 for target in [PLEXUS, OTHER] {
15158 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
15159 let goodbye = module_rx
15160 .try_recv()
15161 .expect("the provider sees the drain")
15162 .frame;
15163 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15164 assert!(module_rx.try_recv().is_err());
15165 }
15166 let rebound = rig
15167 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15168 .await;
15169 assert_eq!(
15170 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
15171 Some("flow:8")
15172 );
15173 }
15174
15175 #[tokio::test]
15180 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
15181 let mut rig = rig().await;
15182 rig.sync(vec![session(1)]).await;
15183 let mut cotenant = rig
15184 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15185 .await;
15186
15187 let mut changed = session(1);
15188 changed.child_owners.push(Principal::Reserved {
15189 module_id: MAGIC.to_string(),
15190 });
15191 let mut ended = None;
15192 for (code, next) in [
15193 (error_codes::SCOPE_CHANGED, vec![changed]),
15194 (error_codes::SCOPE_ENDED, Vec::new()),
15195 ] {
15196 let relayed = rig
15197 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15198 .await;
15199 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
15200 ended = Some(next.is_empty());
15201 rig.sync(next).await;
15202 rig.ack(&relayed).await;
15203 let replies = relayed.task.await.unwrap();
15204 assert_eq!(replies.len(), 1, "{replies:?}");
15205 assert_eq!(parse_error(&replies[0])["code"], code);
15206 assert_eq!(
15207 subc_protocol::error_codes::is_retryable_route_open(code),
15208 code == error_codes::SCOPE_CHANGED
15209 );
15210 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15211 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15213 let mut goodbyes = Vec::new();
15216 while let Ok(outbound) = plexus_rx.try_recv() {
15217 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
15218 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
15219 }
15220 assert!(
15221 goodbyes.contains(&(bind_channel, bind_epoch)),
15222 "{goodbyes:?}"
15223 );
15224 assert!(rig
15225 .handler
15226 .registry
15227 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
15228 .unwrap()
15229 .is_some());
15230 }
15231 assert_eq!(ended, Some(true));
15232 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
15235 }
15236
15237 #[tokio::test]
15240 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
15241 struct Case {
15242 name: &'static str,
15243 change: fn(&mut ScopeRecord),
15244 closed: [Option<RouteCloseReason>; 4],
15247 }
15248 use RouteCloseReason::*;
15249 let cases = [
15250 Case {
15251 name: "a carrier entry removed",
15252 change: |r| {
15253 r.carriers.retain(|c| {
15254 c.principal
15255 != Principal::Reserved {
15256 module_id: AFT.to_string(),
15257 }
15258 })
15259 },
15260 closed: [None, Some(ScopeCarrierRemoved), None, None],
15261 },
15262 Case {
15263 name: "a target removed from a carrier",
15264 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
15265 closed: [None, None, None, Some(ScopeCarrierRemoved)],
15266 },
15267 Case {
15268 name: "a bare carrier narrowed to targets",
15269 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
15270 closed: [None, Some(ScopeCarrierRemoved), None, None],
15271 },
15272 Case {
15273 name: "delegates turned off",
15274 change: |r| r.attributes.delegates = false,
15275 closed: [Some(ScopeDelegationChanged); 4],
15276 },
15277 Case {
15278 name: "agent_id changed",
15279 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
15280 closed: [Some(ScopeDelegationChanged); 4],
15281 },
15282 Case {
15283 name: "a carrier added, child owners changed, the record re-sent",
15284 change: |r| {
15285 r.carriers.push(carrier(MAGIC, None));
15286 r.child_owners.push(Principal::Reserved {
15287 module_id: MAGIC.to_string(),
15288 });
15289 },
15290 closed: [None; 4],
15291 },
15292 Case {
15293 name: "a target added",
15294 change: |r| {
15295 r.carriers[1]
15296 .targets
15297 .as_mut()
15298 .unwrap()
15299 .push("third".to_string())
15300 },
15301 closed: [None; 4],
15302 },
15303 Case {
15304 name: "delegates turned on",
15305 change: |r| r.attributes.delegates = true,
15306 closed: [None; 4],
15307 },
15308 ];
15309 for case in cases {
15310 let mut rig = rig().await;
15311 rig.sync(vec![session(1)]).await;
15312 let scope = || Some(rig_selector("s", Some(1)));
15313 let mut routes = [
15314 rig.bound(Some(OWNER), PLEXUS, scope()).await,
15315 rig.bound(Some(AFT), PLEXUS, scope()).await,
15316 rig.bound(Some(BROCA), PLEXUS, scope()).await,
15317 rig.bound(Some(BROCA), OTHER, scope()).await,
15318 ];
15319 let mut record = session(1);
15320 (case.change)(&mut record);
15321 rig.sync(vec![record]).await;
15322 for (index, expected) in case.closed.iter().enumerate() {
15323 let route = &mut routes[index];
15324 match expected {
15325 Some(reason) => {
15326 assert!(!rig.live(route), "{}: route {index} still live", case.name);
15327 assert_eq!(
15328 route.closed_reason(),
15329 *reason,
15330 "{}: route {index}",
15331 case.name
15332 );
15333 }
15334 None => {
15335 assert!(rig.live(route), "{}: route {index} closed", case.name);
15336 assert!(
15337 route.untouched(),
15338 "{}: route {index} was told something",
15339 case.name
15340 );
15341 }
15342 }
15343 }
15344 }
15345 }
15346
15347 #[tokio::test]
15348 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
15349 for next in [Vec::new(), vec![session(2)]] {
15351 let mut rig = rig().await;
15352 rig.sync(vec![session(1)]).await;
15353 let mut route = rig
15354 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15355 .await;
15356 rig.sync(next).await;
15357 assert!(!rig.live(&route));
15358 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
15359 }
15360
15361 let mut rig = rig().await;
15364 let mut child = session(1);
15365 child.scope_ref = "child".to_string();
15366 child.kind = ScopeKind::Worker;
15367 child.parent = Some(ScopeParent::new(
15368 Principal::Reserved {
15369 module_id: OWNER.to_string(),
15370 },
15371 "s".to_string(),
15372 1,
15373 ));
15374 rig.sync(vec![session(1), child.clone()]).await;
15375 let mut child_route = rig
15376 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
15377 .await;
15378 assert_eq!(
15379 child_route.stamp().unwrap().parent_state,
15380 Some(ParentState::Linked)
15381 );
15382 rig.sync(vec![child]).await;
15383 assert!(!rig.live(&child_route));
15384 assert_eq!(
15385 child_route.closed_reason(),
15386 RouteCloseReason::ScopeParentEnded
15387 );
15388 }
15389
15390 #[tokio::test]
15391 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
15392 ) {
15393 let mut rig = rig().await;
15394 rig.sync(vec![session(1)]).await;
15395 let mut route = rig
15396 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15397 .await;
15398 let before = rig.forwarding.published_scope_tag(OWNER, "s");
15399 rig.sync(vec![session(1)]).await;
15400 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
15401 assert!(rig.live(&route) && route.untouched());
15402
15403 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
15407 .forwarding
15408 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15409 .unwrap()
15410 else {
15411 panic!("the route is bound");
15412 };
15413 binding.flow.acquire_tagged(9, false).await.unwrap();
15414 let mut widened = session(1);
15415 widened.carriers.push(carrier(MAGIC, None));
15416 rig.sync(vec![widened]).await;
15417 assert!(rig.live(&route) && route.untouched());
15418 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15419 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
15420 assert_eq!(binding.flow.in_flight(), 1);
15423 binding
15424 .flow
15425 .acquire_tagged(10, false)
15426 .await
15427 .expect("the flow is still open");
15428 }
15429
15430 #[tokio::test]
15433 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
15434 let mut rig = rig().await;
15435 rig.sync(vec![session(1)]).await;
15436 let mut on_incumbent = rig
15437 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15438 .await;
15439
15440 let (candidate, _candidate_rx) = wide_ctx(9);
15443 let registration = rig
15444 .handler
15445 .registry
15446 .register_candidate_with_control_ops(
15447 manifest(PLEXUS, PROTOCOL_VERSION),
15448 PROTOCOL_VERSION,
15449 candidate.connection_id,
15450 module_baseline_control_ops(),
15451 )
15452 .unwrap();
15453 rig.forwarding
15454 .register_candidate_module_connection(
15455 candidate.connection_id,
15456 PLEXUS.to_string(),
15457 PROTOCOL_VERSION,
15458 manifest_concurrency(®istration.manifest),
15459 candidate.egress.clone(),
15460 )
15461 .unwrap();
15462 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
15463 rig.handler
15464 .registry
15465 .promote_candidate(PLEXUS)
15466 .unwrap()
15467 .unwrap();
15468 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
15469
15470 rig.sync(Vec::new()).await;
15471 assert!(!rig.live(&on_incumbent));
15472 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
15473 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15474 let goodbye = incumbent_rx
15475 .try_recv()
15476 .expect("the superseded endpoint is told")
15477 .frame;
15478 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15479 }
15480 }
15481}
15482
15483#[cfg(test)]
15484mod concurrency_default_exposure_tests {
15485 use super::*;
15486
15487 fn hello_body(role_json: &str) -> Vec<u8> {
15488 format!(
15489 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
15490 )
15491 .into_bytes()
15492 }
15493
15494 fn manifest_from(body: &[u8]) -> ModuleManifest {
15495 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
15496 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
15497 .expect("manifest parses")
15498 }
15499
15500 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
15501
15502 #[test]
15503 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
15504 let body = hello_body(&format!(
15505 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
15506 ));
15507 let manifest = manifest_from(&body);
15508 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
15511 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
15512 }
15513
15514 #[test]
15515 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
15516 let body = hello_body(&format!(
15517 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
15518 ));
15519 let manifest = manifest_from(&body);
15520 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
15521 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
15522 }
15523
15524 #[test]
15525 fn non_management_roles_are_never_reported() {
15526 let body = hello_body(
15527 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
15528 );
15529 let manifest = manifest_from(&body);
15530 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
15531 }
15532}