1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
26 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
27 },
28 scope::{
29 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
30 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
31 SCOPE_SYNC_OP,
32 },
33 session::{
34 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
35 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
36 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
37 ROLE_VERSIONS_FIELD,
38 },
39 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
40 Priority, RouteTarget, PROTOCOL_VERSION,
41};
42use tokio::time::{timeout_at, Instant};
43use tracing::{debug, info, warn};
44
45use crate::{
46 caller_identity::CallerIdentity,
47 capability_requirements::{
48 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
49 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
50 RuntimeModule,
51 },
52 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
53 forwarding::{
54 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
55 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
56 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
57 },
58 observability::{
59 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
60 },
61 privileged_prompt::{prompt_text, PrivilegedOperation},
62 provenance::{
63 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
64 },
65 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
66 router::{RouteCtx, RouterError},
67 scopes::{HelloLaunchNonces, ScopeTable},
68 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
69 stderr_tail::{CaptureState, TailEntry},
70 supervise::{
71 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
72 SupervisorHandle, SwapHelloAdmission,
73 },
74 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
75};
76
77pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
83
84const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
85const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
86const CAP_PING_PONG: &str = "ping_pong_v1";
87const CAP_SESSION_ATTACH: &str = "session_attach_v1";
88const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
89
90const SUBC_CONTROL_OPS: &[&str] = &[
91 ops::ACTIVITY_RECORD,
92 ops::ACTIVITY_QUERY,
93 ops::ACTIVITY_SUBSCRIBE,
94 ops::SERVER_DESCRIBE,
95 ops::CATALOG_LIST,
96 ops::ROUTE_OPEN,
97 ops::ROUTE_POLL,
98 ops::ROUTE_CLOSING,
99 ops::ROUTE_CLOSED,
100 ops::SUPERVISOR_LIST,
101 ops::SUPERVISOR_RESTART,
102 ops::SUPERVISOR_SWAP,
103 ops::SUPERVISOR_RELOAD,
104 ops::SUPERVISOR_RESCAN,
105 ops::SUPERVISOR_RELEASE_RESERVED,
106 ops::SUPERVISOR_SET_ENABLED,
107 ops::SUPERVISOR_HEALTH_PROBE,
108 ops::SUPERVISOR_HEALTH,
109 ops::SUPERVISOR_STDERR_TAIL,
110 ops::SUPERVISOR_TERMINALS,
111 ops::SUPERVISOR_ROUTES,
112 ops::SUPERVISOR_PROVENANCE,
113 ops::SUPERVISOR_SPAWN_SNAPSHOT,
114 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
115];
116
117const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
118 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
119 "supervisor.live_roots",
120 SCOPE_SYNC_OP,
121 SCOPE_APPLY_OP,
122 SCOPE_DESCRIBE_OP,
123 "operator.confirm",
124];
125
126const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
130
131const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
132
133pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
141
142pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
157
158pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
176
177const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
178const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
179
180fn reload_verdict(
181 configured: &Path,
182 spawned_from: Option<&Path>,
183 image: subc_control::RunningImageAgreement,
184) -> PendingReloadVerdict {
185 let path = match spawned_from {
186 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
187 ReloadPathAgreement::Match
188 }
189 Some(spawned_from) => ReloadPathAgreement::Mismatch {
190 configured: configured.to_path_buf(),
191 spawned_from: spawned_from.to_path_buf(),
192 },
193 None => ReloadPathAgreement::Unavailable {
194 reason: if matches!(
195 image,
196 subc_control::RunningImageAgreement::Unavailable {
197 reason: subc_control::RunningImageUnavailableReason::NotRunning
198 }
199 ) {
200 ReloadPathUnavailableReason::NotRunning
201 } else {
202 ReloadPathUnavailableReason::SpawnedPathUnavailable
203 },
204 },
205 };
206 PendingReloadVerdict {
207 path,
208 image,
209 spec: None,
210 }
211}
212
213fn launch_spec_agreement(
214 configured: &crate::supervise::ModuleSpec,
215 running: bool,
216 spawned: Option<&crate::supervise::LaunchSpecDigest>,
217) -> subc_control::LaunchSpecAgreement {
218 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
219 if !running {
220 return Agreement::Unavailable {
221 reason: Reason::NotRunning,
222 };
223 }
224 match spawned {
225 None => Agreement::Unavailable {
226 reason: Reason::SpawnedSpecUnavailable,
227 },
228 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
229 Agreement::Match
230 }
231 Some(_) => Agreement::Mismatch,
232 }
233}
234
235fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
236 if configured == spawned_from {
237 return true;
238 }
239 #[cfg(windows)]
240 {
241 match (
247 std::fs::canonicalize(configured),
248 std::fs::canonicalize(spawned_from),
249 ) {
250 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
251 _ => false,
252 }
253 }
254 #[cfg(not(windows))]
255 {
256 false
257 }
258}
259
260#[derive(Clone)]
261struct DaemonProvenanceFacts {
262 build: DaemonBuildProvenance,
263 pid: Option<u32>,
264 started_at_ms: Option<u64>,
265 start_clock: Option<crate::clock::StartClock>,
266 executable_path: Option<PathBuf>,
267 executable_identity: Option<SpawnedFileIdentity>,
268 process_start_time: Option<u64>,
269 probe: ExecutableIdentityProbe,
270}
271
272impl Default for DaemonProvenanceFacts {
273 fn default() -> Self {
274 Self {
275 build: DaemonBuildProvenance {
276 build_git_sha: None,
277 build_lock_digest: None,
278 },
279 pid: None,
280 started_at_ms: None,
281 start_clock: None,
282 executable_path: None,
283 executable_identity: None,
284 process_start_time: None,
285 probe: ExecutableIdentityProbe::default(),
286 }
287 }
288}
289
290#[derive(Debug, Clone)]
291struct SupervisorRescanContext {
292 supervisor: Supervisor,
293 config_path: PathBuf,
294 configured_port: Option<u16>,
295 storage_config: Option<crate::daemon_config::StorageConfig>,
296 admission_facts_carrier_module_id: Option<String>,
297 admission_facts_targets: Option<Vec<String>>,
298 scope_authority_owners: Vec<String>,
299 privileged_operator_prompt: PrivilegedOperatorPrompt,
300}
301
302const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
323 "reloading",
324 "supervisor_not_live",
325 "registration_not_active",
326 "no_forwarding_connection",
327 "relay_send_failed",
328];
329
330#[derive(Clone)]
332pub struct ControlHandler {
333 registry: Arc<Registry>,
334 activity: Option<crate::activity::ActivityLedger>,
335 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
336 forwarding: Arc<ForwardingTable>,
337 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
338 supervisor: SupervisorHandle,
339 subc_capabilities: Arc<[String]>,
340 route_bind_relay_timeout: Duration,
344 route_bind_relay_timeouts: BTreeMap<String, Duration>,
348 route_bind_breakers: RouteBindBreakers,
351 route_bind_concurrency: RouteBindConcurrency,
354 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
359 route_bind_breaker_threshold: u32,
361 route_bind_breaker_cooldown: Duration,
363 health_probe_timeout: Duration,
364 storage_config: Option<crate::daemon_config::StorageConfig>,
367 machine_id: Option<crate::machine_id::MachineId>,
371 admission_facts_carrier_module_id: Option<String>,
372 admission_facts_targets: Option<Vec<String>>,
373 scopes: Arc<RwLock<ScopeTable>>,
377 scope_deadline_added: Arc<tokio::sync::Notify>,
381 scope_authority_owners: Vec<String>,
384 privileged_operator_prompt: PrivilegedOperatorPrompt,
386 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
389 #[cfg(test)]
390 caller_lookup: Option<TestCallerLookup>,
391 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
394 rescan: Option<SupervisorRescanContext>,
395 connected_clients: ConnectedClients,
396 counters: DaemonCounters,
397 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
398 daemon_provenance: DaemonProvenanceFacts,
399 #[cfg(test)]
400 control_dispatch_delay: Option<Duration>,
401 #[cfg(test)]
402 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
403 #[cfg(test)]
407 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
408 #[cfg(test)]
414 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
415}
416
417impl fmt::Debug for ControlHandler {
418 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
419 f.debug_struct("ControlHandler")
420 .field("registry", &self.registry)
421 .field("forwarding", &self.forwarding)
422 .field("process_liveness", &self.process_liveness.is_some())
423 .field("supervisor", &self.supervisor)
424 .field("subc_capabilities", &self.subc_capabilities)
425 .finish()
426 }
427}
428
429struct RouteOpenRequest {
430 target: RouteTarget,
431 identity: BindIdentity,
432 consumer_identity: Option<ConsumerIdentity>,
433 consumer_capabilities: Option<Vec<String>>,
434 role_versions: Option<BTreeMap<String, String>>,
435 admission_facts: Option<serde_json::Value>,
436 scope: Option<ScopeSelector>,
437}
438
439#[derive(Clone, Copy)]
440enum RouteOpenPhase {
441 Admission,
442 ProjectRoot,
443 EgressReservation,
444 BindSend,
445 BindAckWait,
446}
447
448struct RouteOpenTimings {
452 module_id: String,
453 principal: String,
454 admission: Duration,
455 project_root: Duration,
456 egress_reservation: Duration,
457 bind_send: Duration,
458 bind_ack_wait: Duration,
459 phase: Option<RouteOpenPhase>,
460 phase_started_at: Instant,
461}
462
463impl RouteOpenTimings {
464 fn new() -> Self {
465 Self {
466 module_id: String::new(),
467 principal: "unresolved".to_string(),
469 admission: Duration::ZERO,
470 project_root: Duration::ZERO,
471 egress_reservation: Duration::ZERO,
472 bind_send: Duration::ZERO,
473 bind_ack_wait: Duration::ZERO,
474 phase: Some(RouteOpenPhase::Admission),
475 phase_started_at: Instant::now(),
476 }
477 }
478
479 fn finish(&mut self) {
480 let elapsed = self.phase_started_at.elapsed();
481 match self.phase.take() {
482 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
483 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
484 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
485 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
486 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
487 None => {}
488 }
489 }
490
491 fn enter(&mut self, phase: RouteOpenPhase) {
492 self.finish();
493 self.phase = Some(phase);
494 self.phase_started_at = Instant::now();
495 }
496}
497
498struct RouteBindReservationGuard {
499 forwarding: Arc<ForwardingTable>,
500 endpoint: ModuleEndpointId,
501 relay_corr: u64,
502 armed: bool,
503}
504
505struct ModuleControlRpcGuard {
506 forwarding: Arc<ForwardingTable>,
507 endpoint: ModuleEndpointId,
508 corr: u64,
509 armed: bool,
510}
511
512impl ModuleControlRpcGuard {
513 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
514 Self {
515 forwarding,
516 endpoint,
517 corr,
518 armed: true,
519 }
520 }
521
522 fn disarm(&mut self) {
523 self.armed = false;
524 }
525}
526
527impl Drop for ModuleControlRpcGuard {
528 fn drop(&mut self) {
529 if self.armed {
530 let _ = self
531 .forwarding
532 .cancel_module_control_rpc(self.endpoint, self.corr);
533 }
534 }
535}
536
537impl RouteBindReservationGuard {
538 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
539 Self {
540 forwarding,
541 endpoint,
542 relay_corr,
543 armed: true,
544 }
545 }
546
547 fn release_and_disarm(&mut self) {
548 if !self.armed {
549 return;
550 }
551 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
552 self.endpoint,
553 self.relay_corr,
554 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
555 ) {
556 send_goodbye_target_best_effort(
557 &self.forwarding.counters(),
558 &target,
559 "canceled route.bind",
560 );
561 }
562 self.armed = false;
563 }
564
565 fn disarm(&mut self) {
566 self.armed = false;
567 }
568}
569
570impl Drop for RouteBindReservationGuard {
571 fn drop(&mut self) {
572 self.release_and_disarm();
573 }
574}
575
576#[derive(Debug, Clone, Default)]
606pub(crate) struct RouteBindBreakers {
607 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
608}
609
610#[derive(Debug, Clone, Default)]
611pub(crate) struct RouteBindConcurrency {
612 modules: Arc<Mutex<HashMap<String, usize>>>,
613}
614
615struct RouteBindConcurrencyGuard {
616 concurrency: RouteBindConcurrency,
617 module_id: String,
618}
619
620impl RouteBindConcurrency {
621 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
624 let mut modules = self
625 .modules
626 .lock()
627 .expect("route.bind concurrency mutex poisoned");
628 let in_flight = modules.entry(module_id.to_string()).or_default();
629 if *in_flight >= limit {
630 return Err(*in_flight);
631 }
632 *in_flight += 1;
633 Ok(RouteBindConcurrencyGuard {
634 concurrency: self.clone(),
635 module_id: module_id.to_string(),
636 })
637 }
638}
639
640impl Drop for RouteBindConcurrencyGuard {
641 fn drop(&mut self) {
642 let mut modules = self
643 .concurrency
644 .modules
645 .lock()
646 .expect("route.bind concurrency mutex poisoned");
647 let remove = {
648 let in_flight = modules
649 .get_mut(&self.module_id)
650 .expect("admitted route.bind has a concurrency entry");
651 *in_flight -= 1;
652 *in_flight == 0
653 };
654 if remove {
655 modules.remove(&self.module_id);
656 }
657 }
658}
659
660#[derive(Debug, Default)]
661struct ModuleBreakerState {
662 consecutive_timeouts: u32,
664 cooldown_until: Option<Instant>,
667 probe_in_flight: Option<Arc<()>>,
672}
673
674enum RouteBindAdmission<'a> {
676 Admitted {
677 guard: RouteBindBreakerGuard<'a>,
678 probe: bool,
681 },
682 Refused {
683 consecutive_timeouts: u32,
684 retry_in: Duration,
688 probe_in_flight: bool,
689 },
690}
691
692struct RouteBindBreakerGuard<'a> {
699 breakers: RouteBindBreakers,
700 module_id: &'a str,
701 probe_token: Option<Arc<()>>,
702 settled: bool,
703}
704
705impl RouteBindBreakerGuard<'_> {
706 fn record_accepted(&mut self) -> bool {
710 self.settled = true;
711 self.breakers.record_accepted(self.module_id)
712 }
713
714 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
717 self.settled = true;
718 self.breakers.record_timeout(
719 self.module_id,
720 self.probe_token.as_ref(),
721 threshold,
722 cooldown,
723 )
724 }
725
726 fn record_inconclusive(&mut self) {
735 self.settled = true;
736 self.breakers
737 .record_inconclusive(self.module_id, self.probe_token.as_ref());
738 }
739}
740
741impl Drop for RouteBindBreakerGuard<'_> {
742 fn drop(&mut self) {
743 if !self.settled {
744 self.breakers
745 .record_inconclusive(self.module_id, self.probe_token.as_ref());
746 }
747 }
748}
749
750struct BreakerOpened {
754 consecutive_timeouts: u32,
755 reopened_after_probe: bool,
758}
759
760impl RouteBindBreakers {
761 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
762 self.modules
763 .lock()
764 .expect("route.bind breaker mutex poisoned")
765 }
766
767 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
770 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
771 probe: probe_token.is_some(),
772 guard: RouteBindBreakerGuard {
773 breakers: self.clone(),
774 module_id,
775 probe_token,
776 settled: false,
777 },
778 };
779
780 let mut modules = self.lock();
781 let Some(state) = modules.get_mut(module_id) else {
782 return admitted(None);
783 };
784 let Some(cooldown_until) = state.cooldown_until else {
785 return admitted(None);
786 };
787 if state.probe_in_flight.is_some() {
788 return RouteBindAdmission::Refused {
789 consecutive_timeouts: state.consecutive_timeouts,
790 retry_in: Duration::ZERO,
791 probe_in_flight: true,
792 };
793 }
794 let now = Instant::now();
795 if now < cooldown_until {
796 return RouteBindAdmission::Refused {
797 consecutive_timeouts: state.consecutive_timeouts,
798 retry_in: cooldown_until - now,
799 probe_in_flight: false,
800 };
801 }
802 let token = Arc::new(());
803 state.probe_in_flight = Some(Arc::clone(&token));
804 admitted(Some(token))
805 }
806
807 fn record_accepted(&self, module_id: &str) -> bool {
808 self.lock()
809 .remove(module_id)
810 .is_some_and(|state| state.cooldown_until.is_some())
811 }
812
813 fn record_timeout(
814 &self,
815 module_id: &str,
816 probe_token: Option<&Arc<()>>,
817 threshold: u32,
818 cooldown: Duration,
819 ) -> Option<BreakerOpened> {
820 let mut modules = self.lock();
821 let state = modules.entry(module_id.to_string()).or_default();
822 let was_open = state.cooldown_until.is_some();
823 let was_probe = Self::owns_probe(state, probe_token);
824 if was_probe {
825 state.probe_in_flight = None;
826 }
827 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
828 if state.consecutive_timeouts < threshold {
829 return None;
830 }
831 state.cooldown_until = Some(Instant::now() + cooldown);
832 Some(BreakerOpened {
833 consecutive_timeouts: state.consecutive_timeouts,
834 reopened_after_probe: was_open && was_probe,
835 })
836 }
837
838 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
839 state
843 .probe_in_flight
844 .as_ref()
845 .zip(token)
846 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
847 }
848
849 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
850 if let Some(state) = self.lock().get_mut(module_id) {
851 if Self::owns_probe(state, probe_token) {
852 state.probe_in_flight = None;
853 }
854 }
855 }
856
857 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
872 self.lock()
873 .remove(module_id)
874 .map(|state| state.consecutive_timeouts)
875 .filter(|discarded| *discarded > 0)
876 }
877
878 fn open_snapshot(&self) -> Option<serde_json::Value> {
885 let now = Instant::now();
886 let modules = self.lock();
887 let open = modules
888 .iter()
889 .filter_map(|(module_id, state)| {
890 let cooldown_until = state.cooldown_until?;
891 Some((
892 module_id.clone(),
893 serde_json::json!({
894 "consecutive_timeouts": state.consecutive_timeouts,
895 "cooldown_remaining_ms":
896 cooldown_until.saturating_duration_since(now).as_millis() as u64,
897 "probe_in_flight": state.probe_in_flight.is_some(),
898 }),
899 ))
900 })
901 .collect::<serde_json::Map<String, serde_json::Value>>();
902 (!open.is_empty()).then_some(serde_json::Value::Object(open))
903 }
904}
905
906impl ControlHandler {
907 pub fn new(registry: Arc<Registry>) -> Self {
908 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
909 }
910
911 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
912 let counters = forwarding.counters();
913 let route_bind_breakers = forwarding.route_bind_breakers();
917 let route_bind_concurrency = forwarding.route_bind_concurrency();
918 let route_outages = forwarding.route_outages();
919 Self {
920 registry,
921 activity: None,
922 activity_streams: Arc::new(Mutex::new(HashMap::new())),
923 forwarding,
924 process_liveness: None,
925 supervisor: SupervisorHandle::new(),
926 subc_capabilities: Arc::from([
927 CAP_MANIFEST_REGISTRATION.to_string(),
928 CAP_CHANNEL_LIFECYCLE.to_string(),
929 CAP_PING_PONG.to_string(),
930 CAP_SESSION_ATTACH.to_string(),
931 CAP_ADMISSION_FACTS_RELAY.to_string(),
932 CAP_SCOPES_V1.to_string(),
933 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
934 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
935 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
936 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
937 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
938 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
939 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
940 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
941 ]),
942 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
943 route_bind_relay_timeouts: BTreeMap::new(),
944 route_bind_breakers,
945 route_bind_concurrency,
946 route_outages,
947 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
948 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
949 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
950 storage_config: None,
951 machine_id: None,
952 admission_facts_carrier_module_id: None,
953 admission_facts_targets: None,
954 scopes: Arc::new(RwLock::new(ScopeTable::new(
955 crate::daemon_config::default_scope_authority_owners(),
956 ))),
957 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
958 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
959 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
960 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
961 #[cfg(test)]
962 caller_lookup: None,
963 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
964 rescan: None,
965 connected_clients: ConnectedClients::new(),
966 counters,
967 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
968 daemon_provenance: DaemonProvenanceFacts::default(),
969 #[cfg(test)]
970 control_dispatch_delay: None,
971 #[cfg(test)]
972 provenance_probe_override: None,
973 #[cfg(test)]
974 route_bind_relay_sent_hold: None,
975 #[cfg(test)]
976 hello_registry_to_forwarding_hold: None,
977 }
978 }
979
980 pub fn with_storage_config(
983 mut self,
984 storage_config: Option<crate::daemon_config::StorageConfig>,
985 ) -> Self {
986 self.storage_config = storage_config;
987 self
988 }
989
990 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
993 self.activity = Some(crate::activity::ActivityLedger::open(
994 run_dir.join("activity.jsonl"),
995 incarnation,
996 ));
997 self.supervisor.set_activity(self.activity.clone());
998 self
999 }
1000
1001 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1004 self.machine_id = machine_id;
1005 self
1006 }
1007
1008 pub fn with_admission_facts_config(
1012 mut self,
1013 carrier_module_id: Option<String>,
1014 targets: Option<Vec<String>>,
1015 ) -> Self {
1016 self.admission_facts_carrier_module_id = carrier_module_id;
1017 self.admission_facts_targets = targets;
1018 self
1019 }
1020
1021 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1025 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1026 self.scope_authority_owners = owners;
1027 self
1028 }
1029
1030 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1032 self.privileged_operator_prompt = mode;
1033 self
1034 }
1035
1036 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1039 self.route_bind_relay_timeout = timeout;
1040 self
1041 }
1042
1043 pub fn with_route_bind_relay_timeouts(
1049 mut self,
1050 timeouts: impl IntoIterator<Item = (String, Duration)>,
1051 ) -> Self {
1052 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1053 self
1054 }
1055
1056 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1062 self.route_bind_relay_timeouts
1063 .get(module_id)
1064 .copied()
1065 .unwrap_or(self.route_bind_relay_timeout)
1066 }
1067
1068 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1076 self.route_bind_breaker_threshold = threshold.max(1);
1077 self.route_bind_breaker_cooldown = cooldown;
1078 self
1079 }
1080
1081 #[cfg(test)]
1082 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1083 self.health_probe_timeout = timeout;
1084 self
1085 }
1086
1087 #[cfg(test)]
1088 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1089 self.control_dispatch_delay = Some(delay);
1090 self
1091 }
1092
1093 #[cfg(test)]
1097 pub(crate) fn with_route_bind_relay_sent_hold(
1098 mut self,
1099 hold: Arc<tokio::sync::Barrier>,
1100 ) -> Self {
1101 self.route_bind_relay_sent_hold = Some(hold);
1102 self
1103 }
1104
1105 #[cfg(test)]
1110 pub(crate) fn with_hello_registry_to_forwarding_hold(
1111 mut self,
1112 hold: Arc<std::sync::Barrier>,
1113 ) -> Self {
1114 self.hello_registry_to_forwarding_hold = Some(hold);
1115 self
1116 }
1117
1118 #[cfg(test)]
1119 fn hold_between_registry_and_forwarding(&self) {
1120 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1121 hold.wait();
1122 hold.wait();
1123 }
1124 }
1125
1126 pub fn with_process_liveness(
1127 mut self,
1128 process_liveness: Arc<dyn ModuleProcessLiveness>,
1129 ) -> Self {
1130 self.process_liveness = Some(process_liveness);
1131 self
1132 }
1133
1134 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1135 self.supervisor = supervisor;
1136 self.supervisor.set_activity(self.activity.clone());
1137 self
1138 }
1139
1140 pub fn with_daemon_provenance(
1141 mut self,
1142 pid: u32,
1143 started_at_ms: u64,
1144 executable_path: Option<PathBuf>,
1145 build_git_sha: Option<String>,
1146 build_lock_digest: Option<String>,
1147 ) -> Self {
1148 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1149 let process_start_time = process_start_time(pid);
1150 self.daemon_provenance = DaemonProvenanceFacts {
1151 build: DaemonBuildProvenance {
1152 build_git_sha,
1153 build_lock_digest,
1154 },
1155 pid: Some(pid),
1156 started_at_ms: Some(started_at_ms),
1157 start_clock: None,
1158 executable_path,
1159 executable_identity,
1160 process_start_time,
1161 probe: ExecutableIdentityProbe::default(),
1162 };
1163 self
1164 }
1165
1166 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1167 self.daemon_provenance.start_clock = Some(clock);
1168 self
1169 }
1170
1171 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1173 self.scopes
1174 .write()
1175 .unwrap_or_else(|poisoned| poisoned.into_inner())
1176 .set_wall_clock(Arc::new(clock));
1177 self
1178 }
1179
1180 #[cfg(test)]
1181 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1182 self.provenance_probe_override = Some(result);
1183 self
1184 }
1185
1186 pub fn with_capability_config(
1190 self,
1191 modules: impl IntoIterator<Item = (String, bool)>,
1192 reserved_capabilities: BTreeMap<String, String>,
1193 ) -> Self {
1194 self.capability_evaluator
1195 .configure(modules, reserved_capabilities);
1196 self
1197 }
1198
1199 pub fn with_supervisor_rescan(
1200 mut self,
1201 supervisor: Supervisor,
1202 config_path: impl Into<PathBuf>,
1203 configured_port: Option<u16>,
1204 ) -> Self {
1205 self.rescan = Some(SupervisorRescanContext {
1206 supervisor,
1207 config_path: config_path.into(),
1208 configured_port,
1209 storage_config: self.storage_config.clone(),
1210 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1211 admission_facts_targets: self.admission_facts_targets.clone(),
1212 scope_authority_owners: self.scope_authority_owners.clone(),
1213 privileged_operator_prompt: self.privileged_operator_prompt,
1214 });
1215 self
1216 }
1217
1218 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1219 self.connected_clients = connected_clients;
1220 self
1221 }
1222
1223 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1224 Arc::clone(&self.forwarding)
1225 }
1226
1227 pub(crate) fn counters(&self) -> DaemonCounters {
1228 self.counters.clone()
1229 }
1230
1231 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1234 tokio::spawn(async move {
1235 loop {
1236 self.capability_evaluator
1237 .wait_for_change_or_deadline()
1238 .await;
1239 self.refresh_capability_requirements();
1240 }
1241 });
1242 }
1243
1244 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1250 tokio::spawn(async move {
1251 loop {
1252 let has_deadlines = self
1253 .scopes
1254 .read()
1255 .unwrap_or_else(|poisoned| poisoned.into_inner())
1256 .has_deadlines();
1257 if !has_deadlines {
1258 self.scope_deadline_added.notified().await;
1259 continue;
1260 }
1261 tokio::time::sleep(Duration::from_secs(1)).await;
1262 if let Err(error) = self.sweep_expired_scopes() {
1263 warn!(%error, "scope expiry sweep failed");
1264 }
1265 }
1266 });
1267 }
1268
1269 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1270 let mut table = self
1271 .scopes
1272 .write()
1273 .unwrap_or_else(|poisoned| poisoned.into_inner());
1274 let swept = table.sweep_expired();
1275 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1276 drop(table);
1277 self.close_scope_drained_routes(drained);
1278 Ok(())
1279 }
1280
1281 fn publish_scope_changes(
1282 &self,
1283 changes: &[crate::scopes::ScopeTagChange],
1284 expired: &[crate::scopes::ScopeExpired],
1285 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1286 if changes.is_empty() {
1287 return Ok(Vec::new());
1288 }
1289 let drained = self
1290 .forwarding
1291 .publish_scope_changes(changes)
1292 .map_err(RouterError::Forwarding)?;
1293 let mut counts = HashMap::new();
1294 for route in &drained {
1295 *counts
1296 .entry((
1297 route.scope.owner.as_str(),
1298 route.scope.scope_ref.as_str(),
1299 route.scope.tag.scope_epoch,
1300 ))
1301 .or_insert(0usize) += 1;
1302 }
1303 for scope in expired {
1304 let routes_closed = counts
1305 .get(&(
1306 scope.owner.as_str(),
1307 scope.scope_ref.as_str(),
1308 scope.scope_epoch,
1309 ))
1310 .copied()
1311 .unwrap_or(0);
1312 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1313 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1314 routes_closed, "scope expired");
1315 }
1316 Ok(drained)
1317 }
1318
1319 fn runtime_capability_snapshot(
1320 &self,
1321 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1322 let runtime = self
1323 .supervisor
1324 .list()
1325 .into_iter()
1326 .map(|module| {
1327 let status = module.status().map_err(|err| {
1328 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1329 })?;
1330 Ok(RuntimeModule {
1331 module_id: status.module_id,
1332 state: status.state,
1333 enabled: status.enabled,
1334 })
1335 })
1336 .collect::<Result<Vec<_>, RouterError>>()?;
1337 let registrations = self.registry.list_registered_modules().map_err(|err| {
1343 RouterError::backend(
1344 0,
1345 0,
1346 format!("failed to list capability registrations: {err}"),
1347 )
1348 })?;
1349 let registrations = registrations
1350 .into_iter()
1351 .map(|registration| RegisteredModule {
1352 module_id: registration.manifest.module_id,
1353 module_version: registration.manifest.module_version,
1354 capabilities: registration.manifest.capabilities,
1355 })
1356 .collect();
1357 Ok((runtime, registrations))
1358 }
1359
1360 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1367 let cached_registration = RegisteredModule {
1368 module_id: registration.manifest.module_id.clone(),
1369 module_version: registration.manifest.module_version.clone(),
1370 capabilities: registration.manifest.capabilities.clone(),
1371 };
1372 if self.capability_evaluator.record_hello(&cached_registration) {
1373 warn!(
1374 module_id = %cached_registration.module_id,
1375 "capability claims drifted from the cached manifest"
1376 );
1377 }
1378 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1379 self.enforce_capability_denies();
1380 }
1381 self.refresh_capability_requirements();
1382 }
1383
1384 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1388 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1389 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1390 self.supervisor.set_swap_promotion_observer(observer);
1391 }
1392
1393 pub fn refresh_capability_requirements(&self) {
1394 match self.runtime_capability_snapshot() {
1395 Ok((runtime, registrations)) => {
1396 log_requirement_events(
1397 self.capability_evaluator
1398 .evaluate_now(&runtime, ®istrations),
1399 );
1400 }
1401 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1402 }
1403 }
1404
1405 fn enforce_capability_denies(&self) {
1409 let registrations = match self.registry.list_registered_modules() {
1410 Ok(snapshot) => snapshot,
1411 Err(err) => {
1412 warn!(error = %err, "failed to read registrations for capability deny census");
1413 return;
1414 }
1415 };
1416 let manifests = registrations
1417 .into_iter()
1418 .map(|registration| {
1419 (
1420 registration.manifest.module_id.clone(),
1421 registration.manifest,
1422 )
1423 })
1424 .collect::<BTreeMap<_, _>>();
1425 let census = match self.forwarding.route_census(None) {
1426 Ok(census) => census,
1427 Err(err) => {
1428 warn!(error = %err, "failed to read route census for capability deny enforcement");
1429 return;
1430 }
1431 };
1432
1433 for (target_module_id, routes) in census {
1434 let Some(target_manifest) = manifests.get(&target_module_id) else {
1435 continue;
1436 };
1437 let mut closed_routes = Vec::new();
1438 let mut module_goodbyes = Vec::new();
1439 for route in routes {
1440 let Principal::Reserved {
1441 module_id: opening_module_id,
1442 } = &route.principal
1443 else {
1444 continue;
1445 };
1446 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1447 continue;
1448 };
1449 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1450 continue;
1451 };
1452
1453 match self.forwarding.release_client_route(
1454 route.goodbye_target.connection_id,
1455 route.goodbye_target.channel,
1456 route.goodbye_target.epoch,
1457 ) {
1458 Ok(RouteRelease::Removed(module_goodbye)) => {
1459 warn!(
1460 opening_module_id,
1461 target_module_id,
1462 capability,
1463 "force-closing route because an attested capability deny edge now matches"
1464 );
1465 closed_routes.push(route);
1466 module_goodbyes.push(module_goodbye);
1467 }
1468 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1469 Err(err) => warn!(
1470 opening_module_id,
1471 target_module_id,
1472 capability,
1473 error = %err,
1474 "failed to force-close capability-denied route"
1475 ),
1476 }
1477 }
1478
1479 if closed_routes.is_empty() {
1480 continue;
1481 }
1482 send_route_control_pushes(
1483 &self.forwarding,
1484 closed_routes,
1485 ClientControlPush::RouteClosed {
1486 module_id: target_module_id,
1487 channels: Vec::new(),
1488 reason: RouteCloseReason::CapabilityDenied,
1489 drained: false,
1490 abandoned: 0,
1491 excluded_subscriptions: 0,
1492 terminal: Some(false),
1493 },
1494 );
1495 self.emit_route_goodbyes(module_goodbyes);
1496 }
1497 }
1498
1499 fn not_ready_reason(
1505 &self,
1506 registration: &crate::registry::ModuleRegistration,
1507 ) -> Option<NotReadyReason> {
1508 if !registration.ready {
1509 return Some(NotReadyReason {
1510 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1511 capability: None,
1512 });
1513 }
1514 self.first_unprovided_required_capability(registration)
1515 .map(|capability| NotReadyReason {
1516 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1517 capability: Some(capability),
1518 })
1519 }
1520
1521 fn first_unprovided_required_capability(
1539 &self,
1540 registration: &crate::registry::ModuleRegistration,
1541 ) -> Option<String> {
1542 let required = registration
1543 .manifest
1544 .capabilities
1545 .iter()
1546 .flat_map(|declarations| declarations.requires.iter())
1547 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1548 .map(|requirement| requirement.capability.as_str())
1549 .collect::<BTreeSet<_>>();
1550 if required.is_empty() {
1551 return None;
1552 }
1553 let module_id = registration.manifest.module_id.as_str();
1554 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1555 if required
1556 .iter()
1557 .any(|capability| verdict(capability).is_none())
1558 {
1559 self.refresh_capability_requirements();
1560 }
1561 required
1562 .into_iter()
1563 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1564 .map(str::to_string)
1565 }
1566
1567 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1568 self.capability_evaluator
1569 .statuses()
1570 .into_iter()
1571 .map(capability_requirement_status)
1572 .collect()
1573 }
1574
1575 fn deregister_connection(
1582 &self,
1583 connection_id: ConnectionId,
1584 reason: RegistrationEndReason,
1585 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1586 self.registry
1587 .deregister_connection_with_reason(connection_id, reason)
1588 }
1589
1590 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1591 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1592 return None;
1593 }
1594 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1595 parse_client_control_request(&frame.body)
1596 else {
1597 return None;
1598 };
1599 Some(target_module_id(&target).to_string())
1600 }
1601
1602 pub(crate) fn route_open_capacity_refusal(
1603 &self,
1604 ctx: &RouteCtx,
1605 frame: &Frame,
1606 target_module_id: &str,
1607 in_flight: usize,
1608 limit: usize,
1609 ) -> Result<Frame, RouterError> {
1610 self.route_open_admission_refusal_frame(
1611 ctx,
1612 frame,
1613 target_module_id,
1614 "open_admission_full",
1615 (in_flight, limit),
1616 format!(
1617 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1618 ),
1619 )
1620 }
1621
1622 fn route_open_target_capacity_refusal(
1623 &self,
1624 ctx: &RouteCtx,
1625 frame: &Frame,
1626 target_module_id: &str,
1627 in_flight: usize,
1628 ) -> Result<Frame, RouterError> {
1629 self.route_open_admission_refusal_frame(
1630 ctx,
1631 frame,
1632 target_module_id,
1633 "target_binds_full",
1634 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1635 format!(
1636 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1637 ),
1638 )
1639 }
1640
1641 fn route_open_admission_refusal_frame(
1649 &self,
1650 ctx: &RouteCtx,
1651 frame: &Frame,
1652 target_module_id: &str,
1653 reason: &'static str,
1654 (in_flight, limit): (usize, usize),
1655 message: impl Into<String>,
1656 ) -> Result<Frame, RouterError> {
1657 let code = error_codes::TARGET_UNAVAILABLE;
1658 self.counters.increment_route_open_refused(code);
1659 info!(
1660 target: "control",
1661 code,
1662 reason,
1663 module_id = ?target_module_id,
1664 connection_id = ctx.connection_id.get(),
1665 in_flight,
1666 limit,
1667 "route.open refused"
1668 );
1669 control_error_frame(frame, code, message.into())
1670 }
1671
1672 #[cfg(test)]
1678 pub fn handle_control(
1679 &self,
1680 connection_id: ConnectionId,
1681 frame: Frame,
1682 ) -> Result<Vec<Frame>, RouterError> {
1683 match frame.header.ty {
1684 FrameType::Ping => Ok(vec![pong(&frame)?]),
1685 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1686 FrameType::Goodbye => self.handle_goodbye(connection_id),
1687 ty => Ok(vec![control_error_frame(
1688 &frame,
1689 "unsupported_control_frame",
1690 format!("unsupported channel-0 frame {ty:?}"),
1691 )?]),
1692 }
1693 }
1694
1695 pub async fn handle_control_frame(
1696 &self,
1697 ctx: &RouteCtx,
1698 frame: Frame,
1699 ) -> Result<Vec<Frame>, RouterError> {
1700 self.handle_control_frame_timed(ctx, frame, None).await
1701 }
1702
1703 pub(crate) fn record_caller_endpoints(
1704 &self,
1705 connection: ConnectionId,
1706 endpoints: Option<(SocketAddr, SocketAddr)>,
1707 ) {
1708 let mut callers = self
1709 .caller_endpoints
1710 .lock()
1711 .unwrap_or_else(|p| p.into_inner());
1712 callers.remove(&connection);
1713 if let Some(endpoints) = endpoints {
1714 callers.insert(connection, endpoints);
1715 }
1716 }
1717
1718 #[cfg(test)]
1719 pub(crate) fn with_caller_lookup_for_test(
1720 mut self,
1721 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1722 ) -> Self {
1723 self.caller_lookup = Some(Arc::new(lookup));
1724 self
1725 }
1726
1727 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1728 let endpoints = self
1729 .caller_endpoints
1730 .lock()
1731 .unwrap_or_else(|p| p.into_inner())
1732 .get(&connection)
1733 .copied();
1734 #[cfg(test)]
1735 let lookup = self.caller_lookup.clone();
1736 tokio::task::spawn_blocking(move || {
1737 #[cfg(test)]
1738 if let Some(lookup) = lookup {
1739 return lookup(endpoints);
1740 }
1741 endpoints
1742 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1743 .unwrap_or_default()
1744 })
1745 .await
1746 .unwrap_or_default()
1747 }
1748
1749 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1750 frame.header.channel == 0
1751 && frame.header.ty == FrameType::Request
1752 && self
1753 .forwarding
1754 .module_endpoint_for_connection(ctx.connection_id)
1755 .is_ok_and(|endpoint| endpoint.is_none())
1756 && parse_client_control_request(&frame.body)
1757 .ok()
1758 .is_some_and(|request| privileged_operation(&request).is_some())
1759 }
1760
1761 pub(crate) async fn handle_control_frame_timed(
1762 &self,
1763 ctx: &RouteCtx,
1764 frame: Frame,
1765 dispatch_started_at: Option<StdInstant>,
1766 ) -> Result<Vec<Frame>, RouterError> {
1767 match frame.header.ty {
1768 FrameType::Ping => Ok(vec![pong(&frame)?]),
1769 FrameType::Hello => {
1770 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1771 }
1772 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1773 FrameType::Cancel => {
1774 if self
1777 .forwarding
1778 .operator_confirms()
1779 .cancel(ctx.connection_id, frame.header.corr)
1780 || self
1781 .supervisor
1782 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1783 || self.cancel_activity_subscription(ctx, &frame)?
1784 {
1785 Ok(Vec::new())
1786 } else {
1787 Ok(vec![control_error_frame(
1788 &frame,
1789 "unknown_subscription",
1790 "no subscription has this correlation id",
1791 )?])
1792 }
1793 }
1794 FrameType::Request => {
1795 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1798 if op
1799 .as_ref()
1800 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1801 {
1802 return self.handle_activity_record(ctx, frame).await;
1803 }
1804 if op.as_ref().is_some_and(|probe| {
1805 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1806 }) {
1807 return self.handle_activity_read(ctx, frame).await;
1808 }
1809 if op
1810 .as_ref()
1811 .is_some_and(|probe| probe.op == "operator.confirm")
1812 {
1813 return self.handle_operator_confirm(ctx, frame);
1814 }
1815 if op
1816 .as_ref()
1817 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1818 {
1819 return self.handle_scope_served_by(ctx.connection_id, frame);
1820 }
1821 if self
1822 .forwarding
1823 .module_endpoint_for_connection(ctx.connection_id)
1824 .map_err(RouterError::Forwarding)?
1825 .is_some()
1826 {
1827 if !is_known_module_request_op(&frame.body) {
1828 return Ok(vec![control_error_frame(
1829 &frame,
1830 "unsupported_control_frame",
1831 "module-originated channel-0 REQUEST is not supported",
1832 )?]);
1833 }
1834 let request = match parse_module_control_request_from_module(&frame.body) {
1835 Ok(request) => request,
1836 Err((err, ControlRequestBodyError::UnknownOp)) => {
1837 return Ok(vec![control_error_frame(
1838 &frame,
1839 "unsupported_control_frame",
1840 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1841 )?])
1842 }
1843 Err((err, ControlRequestBodyError::InvalidBody)) => {
1844 return Ok(vec![control_error_frame(
1845 &frame,
1846 "invalid_control_body",
1847 format!("malformed module control body: {err}"),
1848 )?])
1849 }
1850 };
1851 let op = module_control_request_op(&request);
1852 let corr = frame.header.corr;
1853 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1854 let result =
1855 self.handle_module_control_request(ctx.connection_id, frame, request);
1856 log_slow_control_dispatch(
1857 dispatch_started_at,
1858 op,
1859 ctx.connection_id,
1860 corr,
1861 None,
1862 );
1863 return result;
1864 }
1865
1866 if is_known_module_request_op(&frame.body) {
1867 return Ok(vec![control_error_frame(
1868 &frame,
1869 "not_registered",
1870 "catalog.update requires an active module registration owned by this connection",
1871 )?]);
1872 }
1873
1874 let request = match parse_client_control_request(&frame.body) {
1875 Ok(request) => request,
1876 Err((err, ControlRequestBodyError::UnknownOp)) => {
1877 return Ok(vec![control_error_frame(
1878 &frame,
1879 "unknown_control_op",
1880 format!("unknown client control op: {err}"),
1881 )?])
1882 }
1883 Err((err, ControlRequestBodyError::InvalidBody)) => {
1884 return Ok(vec![control_error_frame(
1885 &frame,
1886 "invalid_control_body",
1887 format!("malformed client control body: {err}"),
1888 )?])
1889 }
1890 };
1891 let op = client_control_request_op(&request);
1892 let corr = frame.header.corr;
1893 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1894 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1895 #[cfg(test)]
1896 if let Some(delay) = self.control_dispatch_delay {
1897 tokio::time::sleep(delay).await;
1898 }
1899 let result = self
1900 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1901 .await;
1902 if let Some(timings) = route_open_timings.as_mut() {
1903 timings.finish();
1904 }
1905 log_slow_control_dispatch(
1906 dispatch_started_at,
1907 op,
1908 ctx.connection_id,
1909 corr,
1910 route_open_timings.as_ref(),
1911 );
1912 result
1913 }
1914 FrameType::Push => {
1915 let Some(endpoint) = self
1916 .forwarding
1917 .module_endpoint_for_connection(ctx.connection_id)
1918 .map_err(RouterError::Forwarding)?
1919 else {
1920 return Ok(vec![control_error_frame(
1921 &frame,
1922 "unsupported_control_frame",
1923 "client-originated channel-0 PUSH is not supported",
1924 )?]);
1925 };
1926 self.handle_status_update(endpoint, frame)
1927 }
1928 FrameType::Response | FrameType::Error
1929 if self
1930 .forwarding
1931 .module_endpoint_for_connection(ctx.connection_id)
1932 .map_err(RouterError::Forwarding)?
1933 .is_some() =>
1934 {
1935 self.handle_module_relay_response(ctx.connection_id, frame)
1936 }
1937 ty => Ok(vec![control_error_frame(
1938 &frame,
1939 "unsupported_control_frame",
1940 format!("unsupported channel-0 frame {ty:?}"),
1941 )?]),
1942 }
1943 }
1944
1945 pub fn cleanup_connection(
1946 &self,
1947 connection_id: ConnectionId,
1948 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1949 self.caller_endpoints
1950 .lock()
1951 .unwrap_or_else(|p| p.into_inner())
1952 .remove(&connection_id);
1953 self.forwarding
1954 .operator_confirms()
1955 .connection_closed(connection_id);
1956 self.cleanup_connection_with_end_reason(
1957 connection_id,
1958 RegistrationEndReason::ConnectionClosed,
1959 )
1960 }
1961
1962 fn cleanup_connection_with_end_reason(
1963 &self,
1964 connection_id: ConnectionId,
1965 requested_reason: RegistrationEndReason,
1966 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1967 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1968 self.registry
1969 .get_module_by_connection(connection_id)?
1970 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1971 .and_then(|module| module.registration_end_reason().ok().flatten())
1972 .unwrap_or(requested_reason)
1973 } else {
1974 requested_reason
1975 };
1976 let crash_closed = self
1977 .registry
1978 .get_module_by_connection(connection_id)?
1979 .and_then(|registration| {
1980 self.forwarding
1981 .module_endpoint_for_connection(connection_id)
1982 .ok()
1983 .flatten()
1984 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1985 .map(|routes| (registration.manifest.module_id, routes))
1986 });
1987 let crash_closed = crash_closed.map(|(module_id, routes)| {
1988 let terminal = match self.supervisor.get(&module_id) {
1989 None => false,
1990 Some(module) => match module.will_recover_after_connection_loss() {
1991 Ok(will_recover) => !will_recover,
1992 Err(err) => {
1993 warn!(
1994 %module_id,
1995 error = %err,
1996 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1997 );
1998 false
1999 }
2000 },
2001 };
2002 let (reason, drained) = match self
2006 .forwarding
2007 .daemon_drain_result_for_connection(connection_id)
2008 {
2009 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2010 Ok(None) => (RouteCloseReason::Crash, false),
2011 Err(err) => {
2012 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2013 (RouteCloseReason::Crash, false)
2014 }
2015 };
2016 (module_id, routes, reason, drained, terminal)
2017 });
2018 let registrations = self.deregister_connection(connection_id, end_reason);
2019 let cleanup = if crash_closed.is_some() {
2020 self.forwarding.cleanup_connection_counted(connection_id)
2021 } else {
2022 self.forwarding
2028 .cleanup_connection(connection_id)
2029 .map(|released| crate::forwarding::ConnectionCleanup {
2030 released,
2031 abandoned_relays: 0,
2032 })
2033 };
2034 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2039 let abandoned = cleanup
2040 .as_ref()
2041 .map_or(0, |cleanup| cleanup.abandoned_relays);
2042 send_route_control_pushes(
2043 &self.forwarding,
2044 routes,
2045 ClientControlPush::RouteClosed {
2046 module_id,
2047 channels: Vec::new(),
2048 reason,
2049 drained,
2050 abandoned,
2051 excluded_subscriptions: 0,
2052 terminal: Some(terminal),
2053 },
2054 );
2055 }
2056 if let Ok(cleanup) = cleanup {
2057 self.emit_route_goodbyes(cleanup.released);
2058 }
2059 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2063 crate::supervise::notify_registration_release();
2064 self.capability_evaluator.wake_deadline_loop();
2065 self.refresh_capability_requirements();
2066 }
2067 self.supervisor.remove_spawn_subscribers(connection_id);
2068 self.activity_streams
2069 .lock()
2070 .unwrap_or_else(|p| p.into_inner())
2071 .retain(|(id, _), task| {
2072 if *id == connection_id {
2073 task.abort();
2074 false
2075 } else {
2076 true
2077 }
2078 });
2079 self.hello_launch_nonces
2082 .lock()
2083 .unwrap_or_else(|poisoned| poisoned.into_inner())
2084 .forget(connection_id);
2085 self.scopes
2086 .write()
2087 .unwrap_or_else(|poisoned| poisoned.into_inner())
2088 .release_connection(connection_id);
2089 registrations
2090 }
2091
2092 pub(crate) fn handle_route_goodbye(
2093 &self,
2094 connection_id: ConnectionId,
2095 route_channel: u16,
2096 route_epoch: u32,
2097 ) -> Result<bool, RouterError> {
2098 debug!(
2099 connection_id = connection_id.get(),
2100 route_channel, route_epoch, "handling route GOODBYE"
2101 );
2102 let RouteRelease::Removed(released_route) = self
2103 .forwarding
2104 .release_client_route(connection_id, route_channel, route_epoch)
2105 .map_err(RouterError::Forwarding)?
2106 else {
2107 return Ok(false);
2108 };
2109 self.emit_route_goodbyes(vec![released_route]);
2110 Ok(true)
2111 }
2112
2113 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2114 for released in released_routes {
2115 let frame = match Frame::build_with_version(
2116 released.negotiated_ver,
2117 FrameType::Goodbye,
2118 control_flags(),
2119 released.channel,
2120 released.epoch,
2121 0,
2122 Vec::new(),
2123 ) {
2124 Ok(frame) => frame,
2125 Err(err) => {
2126 warn!(
2127 route_channel = released.channel,
2128 error = %err,
2129 "failed to build route GOODBYE frame"
2130 );
2131 continue;
2132 }
2133 };
2134 if !released.close_on_delivery_failure() {
2135 crate::forwarding::send_module_route_goodbye(
2136 &self.counters,
2137 &released.sink,
2138 frame,
2139 released.module_id.as_deref(),
2140 "client route released",
2141 );
2142 continue;
2143 }
2144 if let Err(err) = released.sink.try_send(frame) {
2145 warn!(
2146 target_connection_id = released.connection_id.get(),
2147 route_channel = released.channel,
2148 error = %err,
2149 "route GOODBYE was not delivered to client; closing target connection"
2150 );
2151 if self
2152 .forwarding
2153 .escalate_client_delivery_failure(
2154 released.connection_id,
2155 released.channel,
2156 released.epoch,
2157 CloseReason::new(
2158 "route_goodbye_delivery_failed",
2159 format!(
2160 "failed to enqueue route GOODBYE for channel {}: {err}",
2161 released.channel
2162 ),
2163 ),
2164 crate::forwarding::UndeliveredFrame {
2165 module_id: released.module_id.as_deref(),
2166 sink: &released.sink,
2167 },
2168 )
2169 .unwrap_or(false)
2170 {
2171 self.counters.increment_goodbye_relay_client_failed();
2172 }
2173 }
2174 }
2175 }
2176
2177 fn send_abandoned_route_bind_goodbye(
2189 &self,
2190 module_sink: &crate::FrameSink,
2191 negotiated_ver: u8,
2192 module_channel: u16,
2193 module_epoch: u32,
2194 ) {
2195 let frame = match Frame::build_with_version(
2196 negotiated_ver,
2197 FrameType::Goodbye,
2198 control_flags(),
2199 module_channel,
2200 module_epoch,
2201 0,
2202 Vec::new(),
2203 ) {
2204 Ok(frame) => frame,
2205 Err(err) => {
2206 warn!(
2207 route_channel = module_channel,
2208 error = %err,
2209 "failed to build GOODBYE for abandoned route.bind"
2210 );
2211 return;
2212 }
2213 };
2214 crate::forwarding::send_module_route_goodbye(
2215 &self.counters,
2216 module_sink,
2217 frame,
2218 None,
2219 "abandoned route.bind",
2220 );
2221 }
2222
2223 fn handle_hello(
2224 &self,
2225 connection_id: ConnectionId,
2226 sink: Option<crate::FrameSink>,
2227 frame: Frame,
2228 ) -> Result<Vec<Frame>, RouterError> {
2229 debug!(
2230 connection_id = connection_id.get(),
2231 corr = frame.header.corr,
2232 "handling HELLO"
2233 );
2234 if self
2238 .registry
2239 .get_module_by_connection(connection_id)
2240 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2241 .is_some()
2242 {
2243 return Ok(vec![control_error_frame(
2244 &frame,
2245 "invalid_hello",
2246 "connection is already registered as a module",
2247 )?]);
2248 }
2249 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2250 Ok(value) => value,
2251 Err(err) => {
2252 return Ok(vec![control_error_frame(
2253 &frame,
2254 "invalid_hello",
2255 format!("malformed HELLO body: {err}"),
2256 )?])
2257 }
2258 };
2259 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2260 return Ok(vec![control_error_frame(
2261 &frame,
2262 "invalid_capability_grammar",
2263 err.to_string(),
2264 )?]);
2265 }
2266 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2267 return Ok(vec![control_error_frame(
2268 &frame,
2269 "invalid_manifest",
2270 err.to_string(),
2271 )?]);
2272 }
2273 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2274 return Ok(vec![control_error_frame(
2275 &frame,
2276 "invalid_event_declaration",
2277 err.to_string(),
2278 )?]);
2279 }
2280 if let Some(provenance) = hello_value
2281 .get("manifest")
2282 .and_then(|manifest| manifest.get("provenance"))
2283 {
2284 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2285 return Ok(vec![control_error_frame(
2286 &frame,
2287 "invalid_manifest",
2288 format!("malformed manifest provenance: {err}"),
2289 )?]);
2290 }
2291 }
2292 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2293 Ok(hello) => hello,
2294 Err(err) => {
2295 return Ok(vec![control_error_frame(
2296 &frame,
2297 "invalid_hello",
2298 format!("malformed HELLO body: {err}"),
2299 )?])
2300 }
2301 };
2302
2303 if hello.protocol_ver != hello.manifest.protocol_ver {
2304 return Ok(vec![control_error_frame(
2305 &frame,
2306 "invalid_manifest",
2307 format!(
2308 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2309 hello.protocol_ver, hello.manifest.protocol_ver
2310 ),
2311 )?]);
2312 }
2313
2314 if hello.manifest.module_id.trim().is_empty() {
2315 return Ok(vec![control_error_frame(
2316 &frame,
2317 "invalid_manifest",
2318 "manifest module_id must not be empty",
2319 )?]);
2320 }
2321
2322 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2323 Ok(negotiated_ver) => negotiated_ver,
2324 Err(message) => {
2325 return Ok(vec![control_error_frame(
2326 &frame,
2327 "version_unsupported",
2328 message,
2329 )?])
2330 }
2331 };
2332
2333 let swap_admission = self
2341 .supervisor
2342 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2343 if swap_admission == SwapHelloAdmission::Refused {
2344 warn!(
2345 module_id = %hello.manifest.module_id,
2346 connection_id = connection_id.get(),
2347 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2348 );
2349 return Ok(vec![control_error_frame(
2350 &frame,
2351 "swap_token_invalid",
2352 format!(
2353 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2354 hello.manifest.module_id
2355 ),
2356 )?]);
2357 }
2358 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2359
2360 if let Some(rejection) = (!swap_candidate)
2368 .then(|| {
2369 self.supervisor.reserved_hello_rejection(
2370 &hello.manifest.module_id,
2371 hello.launch_nonce.as_deref(),
2372 )
2373 })
2374 .flatten()
2375 {
2376 let message = match rejection {
2377 ReservedHelloRejection::Exact { module_id } => format!(
2378 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2379 ),
2380 ReservedHelloRejection::Prefix {
2381 prefix,
2382 owner_module_id,
2383 } => format!(
2384 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2385 hello.manifest.module_id
2386 ),
2387 };
2388 return Ok(vec![control_error_frame(
2389 &frame,
2390 "reserved_module",
2391 message,
2392 )?]);
2393 }
2394
2395 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2396 &hello.manifest.module_id,
2397 hello.manifest.capabilities.as_ref(),
2398 );
2399 if let Some(refusal) = reserved_capability_refusals.first() {
2400 let capability = refusal.capability.clone();
2401 let bound_module = refusal.claimants[0].clone();
2402 log_duplicate_claim_events(reserved_capability_refusals);
2403 return Ok(vec![control_error_frame(
2404 &frame,
2405 "reserved_capability",
2406 format!(
2407 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2408 capability, bound_module, hello.manifest.module_id
2409 ),
2410 )?]);
2411 }
2412
2413 if self
2416 .forwarding
2417 .connection_has_client_routes(connection_id)
2418 .map_err(RouterError::Forwarding)?
2419 {
2420 return Ok(vec![control_error_frame(
2421 &frame,
2422 "invalid_hello",
2423 "connection has open client routes and cannot also register as a module",
2424 )?]);
2425 }
2426
2427 self.hello_launch_nonces
2432 .lock()
2433 .unwrap_or_else(|poisoned| poisoned.into_inner())
2434 .record(connection_id, hello.launch_nonce.as_deref());
2435 let control_ops = effective_module_control_ops(hello.control_ops);
2436 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2439 if swap_candidate {
2440 return self.register_swap_candidate(
2441 connection_id,
2442 sink,
2443 &frame,
2444 hello.manifest,
2445 negotiated_ver,
2446 control_ops,
2447 hello_ack,
2448 );
2449 }
2450 let registration = match self.registry.register_unpublished_with_control_ops(
2457 hello.manifest,
2458 negotiated_ver,
2459 connection_id,
2460 control_ops,
2461 ) {
2462 Ok(registration) => registration,
2463 Err(RegistryError::DuplicateModuleId { module_id }) => {
2464 return Ok(vec![control_error_frame(
2465 &frame,
2466 "duplicate_module_id",
2467 format!(
2468 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2469 ),
2470 )?])
2471 }
2472 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2473 return Ok(vec![control_error_frame(
2474 &frame,
2475 "invalid_module_id",
2476 err.to_string(),
2477 )?])
2478 }
2479 Err(err) => {
2480 return Ok(vec![control_error_frame(
2481 &frame,
2482 "registry_error",
2483 err.to_string(),
2484 )?])
2485 }
2486 };
2487 #[cfg(test)]
2488 self.hold_between_registry_and_forwarding();
2489
2490 let reply = if let Some(sink) = sink {
2491 let concurrency = manifest_concurrency(®istration.manifest);
2507 if let Err(err) = self.forwarding.register_module_connection_acked(
2508 connection_id,
2509 registration.manifest.module_id.clone(),
2510 negotiated_ver,
2511 concurrency,
2512 sink,
2513 hello_ack,
2514 ) {
2515 if matches!(
2520 self.deregister_connection(
2521 connection_id,
2522 RegistrationEndReason::RegistrationFailed,
2523 ),
2524 Ok(r) if !r.is_empty()
2525 ) {
2526 crate::supervise::notify_registration_release();
2527 }
2528 return Ok(vec![control_error_frame(
2529 &frame,
2530 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2531 "invalid_hello"
2532 } else {
2533 forwarding_error_code(&err)
2534 },
2535 err.to_string(),
2536 )?]);
2537 }
2538 Vec::new()
2539 } else {
2540 vec![hello_ack]
2543 };
2544
2545 if !self
2550 .registry
2551 .publish_registration(connection_id)
2552 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2553 {
2554 debug!(
2555 module_id = %registration.manifest.module_id,
2556 connection_id = connection_id.get(),
2557 "registration was removed before it could be published"
2558 );
2559 }
2560
2561 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2570 info!(
2571 module_id = %registration.manifest.module_id,
2572 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2573 );
2574 }
2575
2576 self.apply_registration_capabilities(®istration);
2577
2578 self.supervisor
2579 .record_activity_registration(®istration.manifest.module_id);
2580
2581 info!(
2582 module_id = %registration.manifest.module_id,
2583 module_version = %registration.manifest.module_version,
2584 negotiated_ver,
2585 routable_provider = manifest_provides_routable_role(®istration.manifest),
2586 connection_id = connection_id.get(),
2587 "module registered"
2588 );
2589
2590 Ok(reply)
2591 }
2592
2593 #[allow(clippy::too_many_arguments)]
2610 fn register_swap_candidate(
2611 &self,
2612 connection_id: ConnectionId,
2613 sink: Option<crate::FrameSink>,
2614 frame: &Frame,
2615 manifest: ModuleManifest,
2616 negotiated_ver: u8,
2617 control_ops: Vec<String>,
2618 hello_ack: Frame,
2619 ) -> Result<Vec<Frame>, RouterError> {
2620 let module_id = manifest.module_id.clone();
2621 let registration = match self.registry.register_unpublished_candidate_with_control_ops(
2622 manifest,
2623 negotiated_ver,
2624 connection_id,
2625 control_ops,
2626 ) {
2627 Ok(registration) => registration,
2628 Err(RegistryError::DuplicateModuleId { module_id }) => {
2629 return Ok(vec![control_error_frame(
2630 frame,
2631 "duplicate_module_id",
2632 format!(
2633 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2634 ),
2635 )?])
2636 }
2637 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2638 return Ok(vec![control_error_frame(
2639 frame,
2640 "invalid_module_id",
2641 err.to_string(),
2642 )?])
2643 }
2644 Err(err) => {
2645 return Ok(vec![control_error_frame(
2646 frame,
2647 "registry_error",
2648 err.to_string(),
2649 )?])
2650 }
2651 };
2652 #[cfg(test)]
2653 self.hold_between_registry_and_forwarding();
2654 let reply = if let Some(sink) = sink {
2655 let concurrency = manifest_concurrency(®istration.manifest);
2659 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2660 connection_id,
2661 module_id.clone(),
2662 negotiated_ver,
2663 concurrency,
2664 sink,
2665 hello_ack,
2666 ) {
2667 if matches!(
2668 self.deregister_connection(
2669 connection_id,
2670 RegistrationEndReason::RegistrationFailed,
2671 ),
2672 Ok(r) if !r.is_empty()
2673 ) {
2674 crate::supervise::notify_registration_release();
2675 }
2676 return Ok(vec![control_error_frame(
2677 frame,
2678 forwarding_error_code(&err),
2679 err.to_string(),
2680 )?]);
2681 }
2682 Vec::new()
2683 } else {
2684 vec![hello_ack]
2685 };
2686 if !self
2687 .registry
2688 .publish_registration(connection_id)
2689 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2690 {
2691 debug!(
2692 module_id = %module_id,
2693 connection_id = connection_id.get(),
2694 "swap candidate registration was removed before it could be published"
2695 );
2696 }
2697 self.supervisor.mark_swap_candidate_admitted(&module_id);
2698 info!(
2699 module_id = %module_id,
2700 module_version = %registration.manifest.module_version,
2701 negotiated_ver,
2702 ready = registration.ready,
2703 connection_id = connection_id.get(),
2704 "swap candidate registered; not routable until cutover"
2705 );
2706 Ok(reply)
2707 }
2708
2709 fn build_hello_ack(
2710 &self,
2711 frame: &Frame,
2712 negotiated_ver: u8,
2713 module_id: &str,
2714 ) -> Result<Frame, RouterError> {
2715 let ack = ModuleHelloAckBody {
2716 negotiated_ver,
2717 subc_ops: module_subc_ops(),
2718 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2719 storage: self
2720 .storage_config
2721 .as_ref()
2722 .map(|cfg| cfg.descriptor_for(module_id)),
2723 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2724 };
2725 let body = serde_json::to_vec(&ack).map_err(|err| {
2726 RouterError::backend(
2727 0,
2728 frame.header.corr,
2729 format!("failed to encode HELLO_ACK: {err}"),
2730 )
2731 })?;
2732
2733 Frame::build_with_version(
2734 negotiated_ver,
2735 FrameType::HelloAck,
2736 control_flags(),
2737 0,
2738 0,
2739 frame.header.corr,
2740 body,
2741 )
2742 .map_err(RouterError::FrameBuild)
2743 }
2744
2745 async fn handle_client_control_request(
2746 &self,
2747 ctx: &RouteCtx,
2748 frame: Frame,
2749 request: ClientControlRequest,
2750 route_open_timings: &mut Option<RouteOpenTimings>,
2751 ) -> Result<Vec<Frame>, RouterError> {
2752 if let Some(operation) = privileged_operation(&request) {
2753 let caller = self.resolve_caller(ctx.connection_id).await;
2754 let op = client_control_request_op(&request);
2755 let text = prompt_text(operation, &caller);
2756 match self.privileged_operator_prompt {
2757 PrivilegedOperatorPrompt::Log => {
2758 info!(target: "subc_daemon::privileged_prompt", operation = op,
2759 target = ?operation, caller = %caller.description(), summary = %text,
2760 "privileged request would prompt the operator");
2761 }
2762 PrivilegedOperatorPrompt::Enforce => {
2763 let key =
2764 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2765 let outcome = self
2766 .forwarding
2767 .operator_confirms()
2768 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2769 .await;
2770 if outcome.code != "confirmed" {
2771 return Ok(vec![outcome.frame(&frame)]);
2772 }
2773 }
2774 }
2775 }
2776 match request {
2777 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2778 ClientControlRequest::CatalogList { module_id } => {
2779 self.handle_catalog_list(frame, module_id)
2780 }
2781 ClientControlRequest::RouteOpen {
2782 target,
2783 identity,
2784 consumer_identity,
2785 consumer_capabilities,
2786 role_versions,
2787 admission_facts,
2788 scope,
2789 } => {
2790 self.handle_route_open(
2791 ctx,
2792 frame,
2793 RouteOpenRequest {
2794 target,
2795 identity,
2796 consumer_identity,
2797 consumer_capabilities,
2798 role_versions,
2799 admission_facts,
2800 scope,
2801 },
2802 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2803 )
2804 .await
2805 }
2806 ClientControlRequest::RoutePoll {
2807 route_channel,
2808 route_epoch,
2809 kind,
2810 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2811 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2812 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2813 self.handle_supervisor_spawn_snapshot(frame)
2814 }
2815 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2816 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2817 }
2818 ClientControlRequest::SupervisorRestart {
2819 module_id,
2820 drain_timeout_ms,
2821 } => {
2822 self.log_supervisor_request_received(
2823 ctx,
2824 frame.header.corr,
2825 ops::SUPERVISOR_RESTART,
2826 Some(&module_id),
2827 None,
2828 )?;
2829 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2830 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2831 .await
2832 }
2833 ClientControlRequest::SupervisorSwap {
2834 module_id,
2835 ready_timeout_ms,
2836 } => {
2837 self.log_supervisor_request_received(
2838 ctx,
2839 frame.header.corr,
2840 ops::SUPERVISOR_SWAP,
2841 Some(&module_id),
2842 None,
2843 )?;
2844 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2845 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2846 .await
2847 }
2848 ClientControlRequest::SupervisorReload { module_id } => {
2849 self.log_supervisor_request_received(
2850 ctx,
2851 frame.header.corr,
2852 ops::SUPERVISOR_RELOAD,
2853 Some(&module_id),
2854 None,
2855 )?;
2856 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2857 self.handle_supervisor_reload(frame, module_id, requester)
2858 .await
2859 }
2860 ClientControlRequest::SupervisorRescan { preview } => {
2861 if !preview {
2862 self.log_supervisor_request_received(
2863 ctx,
2864 frame.header.corr,
2865 ops::SUPERVISOR_RESCAN,
2866 None,
2867 None,
2868 )?;
2869 }
2870 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2871 self.handle_supervisor_rescan(frame, preview, requester)
2872 .await
2873 }
2874 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2875 self.log_supervisor_request_received(
2876 ctx,
2877 frame.header.corr,
2878 ops::SUPERVISOR_RELEASE_RESERVED,
2879 Some(&module_id),
2880 None,
2881 )?;
2882 self.handle_supervisor_release_reserved(frame, module_id)
2883 .await
2884 }
2885 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2886 self.log_supervisor_request_received(
2887 ctx,
2888 frame.header.corr,
2889 ops::SUPERVISOR_SET_ENABLED,
2890 Some(&module_id),
2891 Some(enabled),
2892 )?;
2893 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2894 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2895 .await
2896 }
2897 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2898 self.handle_supervisor_health_probe(frame, module_id).await
2899 }
2900 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2901 ClientControlRequest::SupervisorRoutes { module_id } => {
2902 self.handle_supervisor_routes(frame, module_id)
2903 }
2904 ClientControlRequest::SupervisorProvenance { module_id } => {
2905 self.handle_supervisor_provenance(frame, module_id).await
2906 }
2907 ClientControlRequest::SupervisorStderrTail {
2908 module_id,
2909 max_lines,
2910 max_bytes,
2911 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2912 ClientControlRequest::SupervisorTerminals { module_id } => {
2913 self.handle_supervisor_terminals(frame, module_id).await
2914 }
2915 }
2916 }
2917
2918 fn log_supervisor_request_received(
2919 &self,
2920 ctx: &RouteCtx,
2921 corr: u64,
2922 op: &'static str,
2923 module_id: Option<&str>,
2924 enabled: Option<bool>,
2925 ) -> Result<(), RouterError> {
2926 let caller = self.supervisor_requester(ctx, corr)?;
2927 let caller = principal_label(&caller);
2928 info!(
2931 target: "control",
2932 op,
2933 module_id,
2934 enabled,
2935 connection_id = ctx.connection_id.get(),
2936 caller = %caller,
2937 "supervisor request received"
2938 );
2939 Ok(())
2940 }
2941
2942 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2943 Ok(self
2944 .registry
2945 .get_module_by_connection(ctx.connection_id)
2946 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2947 .map(|registration| Principal::Reserved {
2948 module_id: registration.manifest.module_id,
2949 })
2950 .unwrap_or(Principal::Direct))
2951 }
2952
2953 fn handle_module_control_request(
2954 &self,
2955 connection_id: ConnectionId,
2956 frame: Frame,
2957 request: ModuleControlRequestFromModule,
2958 ) -> Result<Vec<Frame>, RouterError> {
2959 match request {
2960 ModuleControlRequestFromModule::CatalogUpdate {
2961 provides,
2962 capabilities,
2963 ready,
2964 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2965 ModuleControlRequestFromModule::LiveRoots {} => {
2966 let registered = self
2967 .registry
2968 .get_module_by_connection(connection_id)
2969 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2970 let Some(registration) = registered else {
2971 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2972 };
2973 let response = self
2974 .forwarding
2975 .live_roots(®istration.manifest.module_id)
2976 .map_err(RouterError::Forwarding)?;
2977 Ok(vec![control_response_body_frame(
2978 &frame,
2979 &response,
2980 "ModuleControlResponseToModule::LiveRoots",
2981 )?])
2982 }
2983 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2984 self.handle_scope_sync(connection_id, frame, generation, scopes)
2985 }
2986 ModuleControlRequestFromModule::ScopeApply {
2987 generation,
2988 upsert,
2989 end,
2990 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2991 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2992 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2993 }
2994 }
2995 }
2996
2997 fn handle_scope_served_by(
2998 &self,
2999 connection_id: ConnectionId,
3000 frame: Frame,
3001 ) -> Result<Vec<Frame>, RouterError> {
3002 let registered = self
3003 .registry
3004 .get_module_by_connection(connection_id)
3005 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3006 let Some(registered) = registered else {
3007 return Ok(vec![control_error_frame(
3008 &frame,
3009 "not_registered",
3010 "scope.served_by requires an active module registration owned by this connection",
3011 )?]);
3012 };
3013 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3014 {
3015 Ok(request) => request,
3016 Err(err) => {
3017 return Ok(vec![control_error_frame(
3018 &frame,
3019 "invalid_control_body",
3020 format!("malformed scope.served_by body: {err}"),
3021 )?])
3022 }
3023 };
3024 let answer = self
3025 .scopes
3026 .read()
3027 .unwrap_or_else(|poisoned| poisoned.into_inner())
3028 .served_by(
3029 ®istered.manifest.module_id,
3030 &request.scope_ref,
3031 request.scope_epoch,
3032 &request.module_id,
3033 );
3034 let reply = subc_control::ScopeServedByReply::new(
3035 answer,
3036 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3037 );
3038 Ok(vec![control_response_body_frame(
3039 &frame,
3040 &reply,
3041 "ScopeServedByReply",
3042 )?])
3043 }
3044
3045 fn handle_operator_confirm(
3046 &self,
3047 ctx: &RouteCtx,
3048 frame: Frame,
3049 ) -> Result<Vec<Frame>, RouterError> {
3050 use crate::operator_confirm::{audit, Outcome};
3051 let registration = self
3052 .registry
3053 .get_module_by_connection(ctx.connection_id)
3054 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3055 let Some(registration) = registration else {
3056 let outcome = Outcome::refusal("not_registered");
3057 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
3058 return Ok(vec![outcome.frame(&frame)]);
3059 };
3060 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3061 Ok(request) => request,
3062 Err(_) => {
3063 let outcome = Outcome::refusal("invalid_control_body");
3064 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
3065 return Ok(vec![outcome.frame(&frame)]);
3066 }
3067 };
3068 let module_id = registration.manifest.module_id;
3069 let nonce = self
3073 .hello_launch_nonces
3074 .lock()
3075 .unwrap_or_else(|p| p.into_inner())
3076 .nonce(ctx.connection_id)
3077 .map(str::to_owned);
3078 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3079 self.supervisor
3080 .spawned_consumer_authorized(&module_id, nonce)
3081 });
3082 let confirms = self.forwarding.operator_confirms();
3083 self.forwarding
3084 .with_operator_route(
3085 ctx.connection_id,
3086 request.route_channel,
3087 request.route_epoch,
3088 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3089 )
3090 .map_err(RouterError::Forwarding)
3091 }
3092
3093 fn handle_scope_sync(
3097 &self,
3098 connection_id: ConnectionId,
3099 frame: Frame,
3100 generation: u64,
3101 scopes: Vec<ScopeRecord>,
3102 ) -> Result<Vec<Frame>, RouterError> {
3103 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3104 }
3105
3106 fn handle_scope_change(
3107 &self,
3108 connection_id: ConnectionId,
3109 frame: Frame,
3110 generation: u64,
3111 scopes: Vec<ScopeRecord>,
3112 end: Option<Vec<ScopeEnd>>,
3113 ) -> Result<Vec<Frame>, RouterError> {
3114 let is_apply = end.is_some();
3115 let Some(registration) = self
3116 .registry
3117 .get_module_by_connection(connection_id)
3118 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3119 else {
3120 return Ok(vec![control_error_frame(
3121 &frame,
3122 "not_registered",
3123 "scope.sync and scope.apply require an active module registration owned by this connection",
3124 )?]);
3125 };
3126 let owner = registration.manifest.module_id;
3127 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3128 let is_current_launch = |connection: ConnectionId| {
3129 self.hello_launch_nonces
3130 .lock()
3131 .unwrap_or_else(|poisoned| poisoned.into_inner())
3132 .presented(connection, current_nonce.as_deref())
3133 };
3134 let mut table = self
3139 .scopes
3140 .write()
3141 .unwrap_or_else(|poisoned| poisoned.into_inner());
3142 let outcome = match end {
3143 Some(end) => table.apply(
3144 &owner,
3145 connection_id,
3146 is_current_launch,
3147 generation,
3148 scopes,
3149 end,
3150 ),
3151 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3152 };
3153 let drained = match &outcome {
3154 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3155 Err(_) => Vec::new(),
3156 };
3157 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3158 drop(table);
3159 if has_deadlines {
3160 self.scope_deadline_added.notify_one();
3161 }
3162 match outcome {
3163 Ok(applied) => {
3164 let counts = ScopeOutcomeCounts::of(&applied.results);
3165 info!(
3166 owner = %owner,
3167 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3168 generation,
3169 records = applied.results.len(),
3170 created = counts.created,
3171 replaced = counts.replaced,
3172 updated = counts.updated,
3173 unchanged = counts.unchanged,
3174 refused = counts.refused,
3175 ended = applied.ended.len(),
3176 tag_changes = applied.tag_changes.len(),
3177 routes_closed = drained.len(),
3178 "scope change accepted"
3179 );
3180 for refused in applied
3186 .results
3187 .iter()
3188 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3189 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3190 {
3191 warn!(
3192 owner = %owner,
3193 generation,
3194 scope_ref = %refused.scope_ref,
3195 scope_epoch = refused.scope_epoch,
3196 code = refused.code.as_deref().unwrap_or(""),
3197 "scope record refused"
3198 );
3199 }
3200 self.close_scope_drained_routes(drained);
3201 let response = if is_apply {
3202 ModuleControlResponseToModule::ScopeApply {
3203 generation,
3204 results: applied.results,
3205 end_results: applied.end_results,
3206 ended: applied.ended,
3207 }
3208 } else {
3209 ModuleControlResponseToModule::ScopeSync {
3210 generation,
3211 results: applied.results,
3212 ended: applied.ended,
3213 }
3214 };
3215 Ok(vec![control_response_body_frame(
3216 &frame,
3217 &response,
3218 "scope change response",
3219 )?])
3220 }
3221 Err(refusal) => {
3222 info!(
3223 owner = %owner,
3224 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3225 generation,
3226 code = refusal.code,
3227 "scope change refused"
3228 );
3229 Ok(vec![control_error_frame(
3230 &frame,
3231 refusal.code,
3232 refusal.message,
3233 )?])
3234 }
3235 }
3236 }
3237
3238 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3244 if drained.is_empty() {
3245 return;
3246 }
3247 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3248 BTreeMap::new();
3249 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3250 for route in drained {
3251 warn!(
3252 module_id = %route.module_id,
3253 reason = ?route.reason,
3254 client_connection_id = route.client.connection_id.get(),
3255 route_channel = route.client.channel,
3256 "closing route because its scope changed"
3257 );
3258 pushes
3259 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3260 .or_insert_with(|| (route.reason, Vec::new()))
3261 .1
3262 .push(EndpointRoute {
3263 goodbye_target: route.client.clone(),
3264 principal: Principal::Unverified,
3265 bound_at: Instant::now(),
3266 draining: false,
3267 drain_reason: None,
3268 });
3269 goodbyes.push(route.module);
3270 goodbyes.push(route.client);
3271 }
3272 for ((module_id, _), (reason, routes)) in pushes {
3273 send_route_control_pushes(
3274 &self.forwarding,
3275 routes,
3276 ClientControlPush::RouteClosed {
3277 module_id,
3278 channels: Vec::new(),
3279 reason,
3280 drained: false,
3281 abandoned: 0,
3282 excluded_subscriptions: 0,
3283 terminal: Some(false),
3284 },
3285 );
3286 }
3287 self.emit_route_goodbyes(goodbyes);
3288 }
3289
3290 fn handle_scope_describe(
3293 &self,
3294 connection_id: ConnectionId,
3295 frame: Frame,
3296 owner: Principal,
3297 scope_ref: String,
3298 ) -> Result<Vec<Frame>, RouterError> {
3299 let registered = self
3300 .registry
3301 .get_module_by_connection(connection_id)
3302 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3303 let Some(registered) = registered else {
3304 return Ok(vec![control_error_frame(
3305 &frame,
3306 "not_registered",
3307 "scope.describe requires an active module registration owned by this connection",
3308 )?]);
3309 };
3310 let mut description = self
3311 .scopes
3312 .read()
3313 .unwrap_or_else(|poisoned| poisoned.into_inner())
3314 .describe(&owner, &scope_ref);
3315 if !registered
3318 .manifest
3319 .capabilities
3320 .as_ref()
3321 .is_some_and(|caps| {
3322 caps.provides
3323 .iter()
3324 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3325 })
3326 {
3327 if let Some(stamp) = description.stamp.as_mut() {
3328 stamp.attributes.on_behalf_of_agent = None;
3329 }
3330 }
3331 let owner_configured = match &owner {
3332 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3337 _ => false,
3338 };
3339 let response = ModuleControlResponseToModule::ScopeDescribe {
3340 status: description.status,
3341 scope_epoch: description.scope_epoch,
3342 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3343 owner_synced: description.owner_synced,
3344 owner_configured,
3345 scope: description.stamp,
3346 };
3347 Ok(vec![control_response_body_frame(
3348 &frame,
3349 &response,
3350 "ModuleControlResponseToModule::ScopeDescribe",
3351 )?])
3352 }
3353
3354 fn handle_catalog_update(
3355 &self,
3356 connection_id: ConnectionId,
3357 frame: Frame,
3358 provides: Vec<ProviderRole>,
3359 capabilities: Option<CapabilityDeclarations>,
3360 ready: Option<bool>,
3361 ) -> Result<Vec<Frame>, RouterError> {
3362 self.refresh_capability_requirements();
3363 let Some(registration) = self
3364 .registry
3365 .get_module_by_connection(connection_id)
3366 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3367 else {
3368 return Ok(vec![control_error_frame(
3369 &frame,
3370 "not_registered",
3371 "catalog.update requires an active module registration owned by this connection",
3372 )?]);
3373 };
3374
3375 if let Some(message) =
3376 catalog_update_frozen_field_message(®istration.manifest, &provides)
3377 {
3378 return Ok(vec![control_error_frame(
3379 &frame,
3380 "catalog_update_frozen_field",
3381 message,
3382 )?]);
3383 }
3384
3385 let mut candidate = registration.manifest.clone();
3386 candidate.provides = provides.clone();
3387 candidate.capabilities = capabilities
3388 .clone()
3389 .or_else(|| registration.manifest.capabilities.clone());
3390 if let Err(err) = candidate.validate_capability_grammar() {
3391 return Ok(vec![control_error_frame(
3392 &frame,
3393 "invalid_capability_grammar",
3394 err.to_string(),
3395 )?]);
3396 }
3397
3398 let mut conflicts = self
3401 .capability_evaluator
3402 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3403 if let Some(conflict) = conflicts.first() {
3404 let message = format!(
3405 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3406 conflict.capability, conflict.claimants[0], candidate.module_id
3407 );
3408 for conflict in &mut conflicts {
3409 conflict.source = DuplicateClaimSource::CatalogUpdate;
3410 }
3411 log_duplicate_claim_events(conflicts);
3412 return Ok(vec![control_error_frame(
3413 &frame,
3414 "reserved_capability",
3415 message,
3416 )?]);
3417 }
3418
3419 let updated = self
3420 .registry
3421 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3422 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3423 if updated.is_none() {
3424 return Ok(vec![control_error_frame(
3425 &frame,
3426 "not_registered",
3427 "catalog.update requires an active module registration owned by this connection",
3428 )?]);
3429 }
3430 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3431 log_duplicate_claim_events(
3432 self.capability_evaluator
3433 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3434 );
3435 }
3436 if capability_census_trigger(
3437 registration.manifest.capabilities.as_ref(),
3438 updated
3439 .as_ref()
3440 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3441 ) {
3442 self.enforce_capability_denies();
3443 }
3444 self.refresh_capability_requirements();
3445
3446 let response = ModuleControlResponseToModule::CatalogUpdate {};
3447 control_response_body_frame(
3448 &frame,
3449 &response,
3450 "ModuleControlResponseToModule::CatalogUpdate",
3451 )
3452 .map(|frame| vec![frame])
3453 }
3454
3455 pub(crate) fn with_activity_ledger_config(
3456 mut self,
3457 run_dir: &Path,
3458 incarnation: String,
3459 config: crate::activity::ActivityConfig,
3460 ) -> Self {
3461 self.activity = Some(crate::activity::ActivityLedger::open_config(
3462 run_dir.join("activity.jsonl"),
3463 incarnation,
3464 config,
3465 ));
3466 self.supervisor.set_activity(self.activity.clone());
3467 self
3468 }
3469
3470 pub(crate) async fn configure_activity(
3471 &self,
3472 config: crate::activity::ActivityConfig,
3473 ) -> Result<(), &'static str> {
3474 match &self.activity {
3475 Some(ledger) => ledger.configure(config).await,
3476 None => Ok(()),
3477 }
3478 }
3479
3480 fn cancel_activity_subscription(
3481 &self,
3482 ctx: &RouteCtx,
3483 frame: &Frame,
3484 ) -> Result<bool, RouterError> {
3485 let task = self
3486 .activity_streams
3487 .lock()
3488 .unwrap_or_else(|p| p.into_inner())
3489 .remove(&(ctx.connection_id, frame.header.corr));
3490 let Some(task) = task else {
3491 return Ok(false);
3492 };
3493 task.abort();
3494 let end = Frame::build_with_version(
3495 response_version(frame),
3496 FrameType::StreamEnd,
3497 control_flags(),
3498 0,
3499 0,
3500 frame.header.corr,
3501 Vec::new(),
3502 )
3503 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3504 let sink = ctx.egress.clone();
3505 tokio::spawn(async move {
3506 let _ = sink.send(end).await;
3507 });
3508 Ok(true)
3509 }
3510
3511 async fn handle_activity_read(
3512 &self,
3513 ctx: &RouteCtx,
3514 frame: Frame,
3515 ) -> Result<Vec<Frame>, RouterError> {
3516 use subc_control::{
3517 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3518 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3519 };
3520 #[derive(Deserialize)]
3521 struct Identity {
3522 #[serde(default)]
3523 consumer_identity: Option<ConsumerIdentity>,
3524 }
3525 let principal = match self
3529 .forwarding
3530 .module_id_for_connection(ctx.connection_id)
3531 .map_err(RouterError::Forwarding)?
3532 {
3533 Some(module_id) => Ok(Principal::Reserved { module_id }),
3534 None => {
3535 let identity: Identity = match serde_json::from_slice(&frame.body) {
3536 Ok(identity) => identity,
3537 Err(_) => {
3538 return Ok(vec![control_error_frame(
3539 &frame,
3540 ACTIVITY_MALFORMED,
3541 "malformed activity read",
3542 )?])
3543 }
3544 };
3545 self.route_open_principal(&frame, identity.consumer_identity)?
3546 }
3547 };
3548 let admitted = matches!(principal, Ok(Principal::Direct))
3549 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3550 if !admitted {
3551 return Ok(vec![control_error_frame(
3552 &frame,
3553 ACTIVITY_READ_FORBIDDEN,
3554 "activity reads require an operator or trusted mirror principal",
3555 )?]);
3556 }
3557 let Some(ledger) = &self.activity else {
3558 return Ok(vec![control_error_frame(
3559 &frame,
3560 ACTIVITY_UNAVAILABLE,
3561 "activity ledger unavailable",
3562 )?]);
3563 };
3564 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3565 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3566 {
3567 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3568 Ok(request) => ledger.query(request).await,
3569 Err(_) => Err(ACTIVITY_MALFORMED),
3570 };
3571 return match result {
3572 Ok(reply) => Ok(vec![control_response_body_frame(
3573 &frame,
3574 &reply,
3575 ops::ACTIVITY_QUERY,
3576 )?]),
3577 Err(code) => Ok(vec![control_error_frame(
3578 &frame,
3579 code,
3580 "activity query refused",
3581 )?]),
3582 };
3583 }
3584 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3585 Ok(request) => request,
3586 Err(_) => {
3587 return Ok(vec![control_error_frame(
3588 &frame,
3589 ACTIVITY_MALFORMED,
3590 "malformed activity subscription",
3591 )?])
3592 }
3593 };
3594 let mut subscription = match ledger.subscribe(request.since).await {
3595 Ok(subscription) => subscription,
3596 Err(code) => {
3597 return Ok(vec![control_error_frame(
3598 &frame,
3599 code,
3600 "activity subscription refused",
3601 )?])
3602 }
3603 };
3604 let key = (ctx.connection_id, frame.header.corr);
3605 let sink = ctx.egress.clone();
3606 let mut streams = self
3607 .activity_streams
3608 .lock()
3609 .unwrap_or_else(|p| p.into_inner());
3610 streams.retain(|_, task| !task.is_finished());
3611 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3612 return Ok(vec![control_error_frame(
3613 &frame,
3614 "duplicate_subscription",
3615 "activity subscription correlation id already in use",
3616 )?]);
3617 }
3618 let task = tokio::spawn(async move {
3619 loop {
3620 let entry = if let Some(entry) = subscription.replay.pop_front() {
3621 Some(entry)
3622 } else {
3623 subscription.receiver.recv().await
3624 };
3625 let Some(entry) = entry else {
3626 break;
3627 };
3628 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3629 return;
3630 };
3631 let Ok(next) = Frame::build_with_version(
3632 response_version(&frame),
3633 FrameType::StreamData,
3634 control_flags(),
3635 0,
3636 0,
3637 frame.header.corr,
3638 body,
3639 ) else {
3640 return;
3641 };
3642 if sink.send(next).await.is_err() {
3643 return;
3644 }
3645 }
3646 if let Some(code) = subscription.terminal_code() {
3649 if let Ok(end) =
3650 control_error_frame(&frame, code, "activity subscriber fell behind")
3651 {
3652 let _ = sink.send(end).await;
3653 }
3654 }
3655 });
3656 streams.insert(key, task.abort_handle());
3657 Ok(Vec::new())
3658 }
3659
3660 async fn handle_activity_record(
3661 &self,
3662 ctx: &RouteCtx,
3663 frame: Frame,
3664 ) -> Result<Vec<Frame>, RouterError> {
3665 use subc_control::{
3666 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3667 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3668 };
3669 let result = async {
3670 let module = self
3671 .forwarding
3672 .module_id_for_connection(ctx.connection_id)
3673 .ok()
3674 .flatten()
3675 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3676 let request: ActivityRecordRequest =
3677 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3678 crate::activity::validate(&module, &request)?;
3679 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3680 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3681 let at_ms = std::time::SystemTime::now()
3682 .duration_since(std::time::UNIX_EPOCH)
3683 .unwrap_or_default()
3684 .as_millis() as u64;
3685 ledger.record(module, request, actor, at_ms).await
3686 }
3687 .await;
3688 match result {
3689 Ok(entry) => Ok(vec![control_response_body_frame(
3690 &frame,
3691 &ActivityRecordResponse { seq: entry.seq },
3692 "activity.record",
3693 )?]),
3694 Err(code) => Ok(vec![control_error_frame(
3695 &frame,
3696 code,
3697 "activity recording refused",
3698 )?]),
3699 }
3700 }
3701
3702 fn activity_actor(
3703 &self,
3704 connection: ConnectionId,
3705 module: &str,
3706 on: Option<&subc_control::ActivityOn>,
3707 ) -> Result<subc_control::ActivityActor, &'static str> {
3708 use crate::forwarding::{DataRoute, DataRouteState};
3709 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3710 let principal = |id: &str| Principal::Reserved {
3711 module_id: id.to_owned(),
3712 };
3713 let mut actor = ActivityActor {
3714 principal: principal(module),
3715 scope_owner: None,
3716 agent_id: None,
3717 run_id: None,
3718 flow_id: None,
3719 project_root: None,
3720 };
3721 let scope = match on {
3722 None => return Ok(actor),
3723 Some(ActivityOn::Scope(scope)) => {
3724 let selector = subc_protocol::scope::ScopeSelector::new(
3725 principal(module),
3726 scope.scope_ref.clone(),
3727 Some(scope.epoch),
3728 );
3729 self.scopes
3730 .read()
3731 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3732 .attribute(&principal(module), module, &selector, true)
3733 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3734 }
3735 Some(ActivityOn::Route(route)) => {
3736 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3737 .forwarding
3738 .lookup_data_route(connection, route.channel, route.epoch)
3739 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3740 else {
3741 return Err(ACTIVITY_ACTOR_UNKNOWN);
3742 };
3743 if binding.module_id != module {
3744 return Err(ACTIVITY_ACTOR_UNKNOWN);
3745 }
3746 actor.principal = binding.principal.clone();
3747 actor.project_root = binding
3748 .project_root
3749 .as_ref()
3750 .map(|root| root.as_path().to_path_buf());
3751 let Some(bound) = &binding.scope else {
3752 return Ok(actor);
3753 };
3754 let owner = principal(&bound.owner);
3755 let selector = subc_protocol::scope::ScopeSelector::new(
3756 owner.clone(),
3757 bound.scope_ref.clone(),
3758 Some(bound.tag.scope_epoch),
3759 );
3760 let admitted = self
3761 .scopes
3762 .read()
3763 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3764 .attribute(&owner, module, &selector, true)
3765 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3766 if admitted.tag != bound.tag {
3767 return Err(ACTIVITY_ACTOR_UNKNOWN);
3768 }
3769 admitted
3770 }
3771 };
3772 actor.scope_owner = Some(scope.stamp.owner);
3773 actor.agent_id = scope.stamp.attributes.agent_id;
3774 actor.run_id = scope.stamp.attributes.run_id;
3775 actor.flow_id = scope.stamp.attributes.flow_id;
3776 Ok(actor)
3777 }
3778
3779 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3780 self.refresh_capability_requirements();
3781 let mut counters = self.counters.snapshot();
3787 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3788 obj.insert(
3789 "activity_unparseable_lines".into(),
3790 ledger.read_errors().into(),
3791 );
3792 obj.insert(
3793 "activity_lifecycle_append_failures".into(),
3794 ledger.lifecycle_failures().into(),
3795 );
3796 obj.insert(
3797 "activity_started_digest_skips".into(),
3798 ledger.lifecycle_digest_skips().into(),
3799 );
3800 }
3801 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3802 self.forwarding.client_route_concentration(),
3803 counters.as_object_mut(),
3804 ) {
3805 obj.insert(
3806 "client_connections_with_routes".into(),
3807 connections_with_routes.into(),
3808 );
3809 obj.insert("max_routes_on_one_connection".into(), max.into());
3810 }
3811 if let (Some(open_breakers), Some(obj)) = (
3817 self.route_bind_breakers.open_snapshot(),
3818 counters.as_object_mut(),
3819 ) {
3820 obj.insert("route_bind_breakers_open".into(), open_breakers);
3821 }
3822 let response = ClientControlResponse::ServerDescribe {
3823 protocol_ver: PROTOCOL_VERSION,
3824 subc_ops: subc_ops(),
3825 capabilities: self.subc_capabilities.as_ref().to_vec(),
3826 connected_clients: self.connected_clients.count(),
3827 counters: Some(counters),
3828 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3829 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3830 capability_requirements: self.capability_requirement_statuses(),
3831 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3832 };
3833 Ok(vec![control_response_body_frame(
3834 &frame,
3835 &response,
3836 "ClientControlResponse::ServerDescribe",
3837 )?])
3838 }
3839
3840 fn handle_catalog_list(
3841 &self,
3842 frame: Frame,
3843 module_id: Option<String>,
3844 ) -> Result<Vec<Frame>, RouterError> {
3845 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3846 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3847 })?;
3848 let entries = modules
3849 .into_iter()
3850 .filter(|registration| {
3851 module_id
3852 .as_deref()
3853 .map(|wanted| registration.manifest.module_id == wanted)
3854 .unwrap_or(true)
3855 })
3856 .map(|registration| {
3857 let not_ready = self.not_ready_reason(®istration);
3858 let roles = registration.manifest.provides;
3859 CatalogEntry::new(
3860 registration.manifest.module_id,
3861 roles,
3862 registration.control_ops,
3863 )
3864 .with_ready(not_ready.is_none())
3865 .with_not_ready(not_ready)
3866 .with_module_version(Some(registration.manifest.module_version))
3867 .with_capabilities(registration.manifest.capabilities)
3868 .with_self_signals(registration.manifest.self_signals)
3869 .with_events(registration.manifest.events)
3870 .with_operation_delivery(registration.manifest.operation_delivery)
3871 })
3872 .collect();
3873 let response = ClientControlResponse::CatalogList {
3874 generation,
3875 modules: entries,
3876 subc_ops: subc_ops(),
3877 };
3878 Ok(vec![control_response_body_frame(
3879 &frame,
3880 &response,
3881 "ClientControlResponse::CatalogList",
3882 )?])
3883 }
3884
3885 fn route_open_principal(
3886 &self,
3887 frame: &Frame,
3888 consumer_identity: Option<ConsumerIdentity>,
3889 ) -> Result<Result<Principal, Frame>, RouterError> {
3890 let Some(consumer_identity) = consumer_identity else {
3891 return Ok(Ok(Principal::Direct));
3892 };
3893
3894 if self.supervisor.spawned_consumer_authorized(
3895 &consumer_identity.module_id,
3896 &consumer_identity.launch_nonce,
3897 ) {
3898 return Ok(Ok(Principal::Reserved {
3899 module_id: consumer_identity.module_id,
3900 }));
3901 }
3902
3903 Ok(Err(control_error_frame(
3904 frame,
3905 "bad_consumer_identity",
3906 format!(
3907 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3908 consumer_identity.module_id
3909 ),
3910 )?))
3911 }
3912
3913 fn route_open_refusal_frame(
3920 &self,
3921 ctx: &RouteCtx,
3922 frame: &Frame,
3923 module_id: &str,
3924 reason: &'static str,
3925 code: &'static str,
3926 message: impl Into<String>,
3927 ) -> Result<Frame, RouterError> {
3928 self.observe_route_open_refusal(ctx, module_id, reason, code);
3929 control_error_frame(frame, code, message.into())
3930 }
3931
3932 fn route_open_breaker_refusal_frame(
3950 &self,
3951 ctx: &RouteCtx,
3952 frame: &Frame,
3953 module_id: &str,
3954 consecutive_timeouts: u32,
3955 retry_in: Duration,
3956 probe_in_flight: bool,
3957 ) -> Result<Frame, RouterError> {
3958 self.counters
3959 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3960 debug!(
3961 target: "control",
3962 code = "module_timeout",
3963 module_id = ?module_id,
3964 connection_id = ctx.connection_id.get(),
3965 consecutive_timeouts,
3966 retry_in_ms = retry_in.as_millis() as u64,
3967 probe_in_flight,
3968 "route.open refused by open bind-relay breaker"
3969 );
3970 let detail = if probe_in_flight {
3977 "one test route is already being tried; retry once it settles".to_string()
3978 } else {
3979 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3980 };
3981 control_error_frame(
3982 frame,
3983 "module_timeout",
3984 format!(
3985 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3986 timed out in a row); {detail}; its established routes are unaffected"
3987 ),
3988 )
3989 }
3990
3991 fn observe_route_open_refusal(
4004 &self,
4005 ctx: &RouteCtx,
4006 module_id: &str,
4007 reason: &'static str,
4008 code: &'static str,
4009 ) {
4010 self.counters.increment_route_open_refused(code);
4011 info!(
4012 target: "control",
4013 code,
4014 reason,
4015 module_id = ?module_id,
4016 connection_id = ctx.connection_id.get(),
4017 "route.open refused"
4018 );
4019 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4020 self.route_outages.record_not_serving(module_id, reason);
4021 }
4022 }
4023
4024 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
4069 self.route_outages.record_accepted(module_id);
4070 self.counters.increment_route_open_accepted(principal);
4071 info!(
4072 target: "control",
4073 principal,
4074 module_id,
4075 connection_id = ctx.connection_id.get(),
4076 "route.open accepted"
4077 );
4078 }
4079
4080 fn supervised_absent_route_open_refusal_frame(
4081 &self,
4082 ctx: &RouteCtx,
4083 frame: &Frame,
4084 module_id: &str,
4085 code: &'static str,
4086 status: &crate::supervise::ModuleStatus,
4087 ) -> Result<Frame, RouterError> {
4088 self.counters.increment_route_open_refused(code);
4089 info!(
4090 target: "control",
4091 code,
4092 reason = "supervised_not_registered",
4093 module_id = ?module_id,
4094 connection_id = ctx.connection_id.get(),
4095 state = %status.state,
4096 enabled = status.enabled,
4097 live = status.live,
4098 "route.open refused"
4099 );
4100 self.route_outages
4104 .record_not_serving(module_id, "supervised_not_registered");
4105 let message = if let Some(pid) = status.waiting_for_previous_pid {
4106 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4107 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4108 })
4109 } else {
4110 format!(
4111 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4112 status.state, status.enabled, status.live
4113 )
4114 };
4115 control_error_frame(frame, code, message)
4116 }
4117
4118 async fn handle_route_open(
4119 &self,
4120 ctx: &RouteCtx,
4121 frame: Frame,
4122 request: RouteOpenRequest,
4123 timings: &mut RouteOpenTimings,
4124 ) -> Result<Vec<Frame>, RouterError> {
4125 let RouteOpenRequest {
4126 target,
4127 mut identity,
4128 consumer_identity,
4129 consumer_capabilities,
4130 role_versions,
4131 admission_facts,
4132 scope,
4133 } = request;
4134 let target_module_id = target_module_id(&target).to_string();
4135 timings.module_id.clone_from(&target_module_id);
4136 if self
4137 .registry
4138 .get_module_by_connection(ctx.connection_id)
4139 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4140 .is_some()
4141 {
4142 return Ok(vec![control_error_frame(
4143 &frame,
4144 "invalid_request",
4145 "module connections cannot open client routes",
4146 )?]);
4147 }
4148 debug!(
4149 connection_id = ctx.connection_id.get(),
4150 corr = frame.header.corr,
4151 module_id = %target_module_id,
4152 "handling route.open"
4153 );
4154
4155 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4161 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4162 self.observe_route_open_refusal(
4163 ctx,
4164 &target_module_id,
4165 "invalid_role_versions",
4166 error_codes::INVALID_REQUEST,
4167 );
4168 return Ok(vec![control_error_body_frame(
4169 &frame,
4170 ErrorBody {
4171 code: error_codes::INVALID_REQUEST.to_string(),
4172 message: error.to_string(),
4173 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4174 },
4175 )?]);
4176 }
4177
4178 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4179 if status.waiting_for_previous_pid.is_some() {
4180 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4181 ctx,
4182 &frame,
4183 &target_module_id,
4184 "module_warming",
4185 &status,
4186 )?]);
4187 }
4188 }
4189
4190 let Some(registration) = self
4209 .registry
4210 .get_module(&target_module_id)
4211 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4212 else {
4213 if let Some((status, warming)) =
4214 self.supervisor_status(&target_module_id, frame.header.corr)?
4215 {
4216 if status.protocol == ModuleProtocol::None {
4223 return Ok(vec![self.route_open_refusal_frame(
4224 ctx,
4225 &frame,
4226 &target_module_id,
4227 "protocol_none",
4228 error_codes::MODULE_NO_PROTOCOL,
4229 format!(
4230 "module_id '{target_module_id}' is declared protocol: none; \
4231 it speaks no subc wire and serves no routes"
4232 ),
4233 )?]);
4234 }
4235 let code = if warming {
4236 "module_warming"
4237 } else {
4238 "target_unavailable"
4239 };
4240 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4241 ctx,
4242 &frame,
4243 &target_module_id,
4244 code,
4245 &status,
4246 )?]);
4247 }
4248 if let Some(removed_ago_ms) =
4249 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4250 {
4251 return Ok(vec![self.route_open_refusal_frame(
4252 ctx,
4253 &frame,
4254 &target_module_id,
4255 "removed",
4256 error_codes::MODULE_REMOVED,
4257 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4258 )?]);
4259 }
4260 return Ok(vec![self.route_open_refusal_frame(
4261 ctx,
4262 &frame,
4263 &target_module_id,
4264 "not_registered",
4265 error_codes::UNKNOWN_MODULE,
4266 format!("module_id '{target_module_id}' is not registered"),
4267 )?]);
4268 };
4269
4270 if !registration.ready {
4274 self.counters
4275 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4276 info!(
4277 target: "control",
4278 code = error_codes::MODULE_WARMING,
4279 module_id = ?target_module_id,
4280 connection_id = ctx.connection_id.get(),
4281 reason = "declared_not_ready",
4282 "route.open refused"
4283 );
4284 self.route_outages
4287 .record_not_serving(&target_module_id, "declared_not_ready");
4288 return Ok(vec![control_error_body_frame(
4289 &frame,
4290 ErrorBody {
4291 code: error_codes::MODULE_WARMING.to_string(),
4292 message: format!(
4293 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4294 ),
4295 detail: Some(serde_json::json!({
4296 "reason": "declared_not_ready"
4297 })),
4298 },
4299 )?]);
4300 }
4301
4302 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4321 self.counters
4322 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4323 info!(
4324 target: "control",
4325 code = error_codes::MODULE_WARMING,
4326 module_id = ?target_module_id,
4327 connection_id = ctx.connection_id.get(),
4328 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4329 capability = %capability,
4330 "route.open refused"
4331 );
4332 return Ok(vec![control_error_body_frame(
4333 &frame,
4334 ErrorBody {
4335 code: error_codes::MODULE_WARMING.to_string(),
4336 message: format!(
4337 "module_id '{target_module_id}' requires capability '{capability}', \
4338 which no registered module provides; retry"
4339 ),
4340 detail: Some(serde_json::json!({
4341 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4342 "capability": capability,
4343 })),
4344 },
4345 )?]);
4346 }
4347
4348 if !target_has_required_role(&target, ®istration.manifest.provides) {
4349 return Ok(vec![self.route_open_refusal_frame(
4350 ctx,
4351 &frame,
4352 &target_module_id,
4353 "role_not_provided",
4354 "target_unavailable",
4355 format!("module_id '{target_module_id}' does not provide the requested target"),
4356 )?]);
4357 }
4358
4359 if registration.state != ChannelState::Active {
4360 return Ok(vec![self.route_open_refusal_frame(
4361 ctx,
4362 &frame,
4363 &target_module_id,
4364 "registration_not_active",
4365 "target_unavailable",
4366 format!("module_id '{target_module_id}' is not active"),
4367 )?]);
4368 }
4369
4370 if self
4371 .forwarding
4372 .module_is_draining(&target_module_id)
4373 .map_err(RouterError::Forwarding)?
4374 {
4375 return Ok(vec![self.route_open_refusal_frame(
4376 ctx,
4377 &frame,
4378 &target_module_id,
4379 "reloading",
4380 "module_reloading",
4381 format!("module_id '{target_module_id}' is reloading"),
4382 )?]);
4383 }
4384
4385 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4386 process_liveness.process_live(&target_module_id) == Some(false)
4387 }) {
4388 if process_liveness.process_replacing(&target_module_id) {
4395 return Ok(vec![self.route_open_refusal_frame(
4396 ctx,
4397 &frame,
4398 &target_module_id,
4399 "reloading",
4400 "module_reloading",
4401 format!("module_id '{target_module_id}' is reloading"),
4402 )?]);
4403 }
4404 return Ok(vec![self.route_open_refusal_frame(
4405 ctx,
4406 &frame,
4407 &target_module_id,
4408 "supervisor_not_live",
4409 "target_unavailable",
4410 format!("module_id '{target_module_id}' is not live"),
4411 )?]);
4412 }
4413
4414 if !self
4415 .forwarding
4416 .has_live_module_connection(&target_module_id)
4417 .map_err(RouterError::Forwarding)?
4418 {
4419 return Ok(vec![self.route_open_refusal_frame(
4420 ctx,
4421 &frame,
4422 &target_module_id,
4423 "no_forwarding_connection",
4424 "target_unavailable",
4425 format!("module_id '{target_module_id}' has no live forwarding connection"),
4426 )?]);
4427 }
4428
4429 if let Some(error) =
4430 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4431 {
4432 self.observe_route_open_refusal(
4433 ctx,
4434 &target_module_id,
4435 "op_not_allowed",
4436 "op_not_allowed",
4437 );
4438 return Ok(vec![error]);
4439 }
4440
4441 let principal = match self.route_open_principal(&frame, consumer_identity)? {
4442 Ok(principal) => principal,
4443 Err(error) => {
4444 self.observe_route_open_refusal(
4445 ctx,
4446 &target_module_id,
4447 "bad_consumer_identity",
4448 "bad_consumer_identity",
4449 );
4450 return Ok(vec![error]);
4451 }
4452 };
4453 timings.principal = principal_label(&principal);
4454
4455 if let Principal::Reserved {
4459 module_id: opening_module_id,
4460 } = &principal
4461 {
4462 if let Some(opening_registration) = self
4463 .registry
4464 .get_module(opening_module_id)
4465 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4466 {
4467 if let Some(capability) =
4468 denied_capability(&opening_registration.manifest, ®istration.manifest)
4469 {
4470 warn!(
4471 opening_module_id,
4472 target_module_id,
4473 capability,
4474 "refusing route.open because an attested capability deny edge matches"
4475 );
4476 return Ok(vec![self.route_open_refusal_frame(
4477 ctx,
4478 &frame,
4479 &target_module_id,
4480 "capability_deny_edge",
4481 "capability_forbidden",
4482 format!(
4483 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4484 ),
4485 )?]);
4486 }
4487 }
4488 }
4489
4490 if admission_facts.is_some() {
4491 let carrier_matches = matches!(
4492 &principal,
4493 Principal::Reserved { module_id }
4494 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4495 );
4496 if !carrier_matches {
4497 return Ok(vec![self.route_open_refusal_frame(
4498 ctx,
4499 &frame,
4500 &target_module_id,
4501 "admission_facts_carrier_not_permitted",
4502 "admission_facts_not_permitted",
4503 "admission facts may only be carried by the configured reserved module",
4504 )?]);
4505 }
4506
4507 let target_allowed = self
4508 .admission_facts_targets
4509 .as_ref()
4510 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4511 if !target_allowed {
4512 return Ok(vec![self.route_open_refusal_frame(
4513 ctx,
4514 &frame,
4515 &target_module_id,
4516 "admission_facts_target_not_listed",
4517 "admission_facts_target_not_allowed",
4518 format!(
4519 "admission facts are not permitted for target module_id '{target_module_id}'"
4520 ),
4521 )?]);
4522 }
4523
4524 }
4528
4529 let (bound_scope, mut scope_stamp) = match &scope {
4535 None => (None, None),
4536 Some(selector) => {
4537 let owner_configured = match &selector.owner {
4538 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4544 _ => false,
4545 };
4546 let mut scopes = self
4547 .scopes
4548 .write()
4549 .unwrap_or_else(|poisoned| poisoned.into_inner());
4550 let admitted = scopes
4551 .admit(&principal, &target_module_id, selector, owner_configured)
4552 .and_then(|admission| {
4553 crate::scopes::check_target_flow_support(
4554 &admission.stamp,
4555 &target_module_id,
4556 registration.manifest.capabilities.as_ref(),
4557 )?;
4558 crate::scopes::check_target_agent_run_support(
4559 &admission.stamp,
4560 &target_module_id,
4561 registration.manifest.capabilities.as_ref(),
4562 )?;
4563 let bound = scopes.reserve_bind(
4564 &admission,
4565 &principal,
4566 ctx.connection_id,
4567 selector.hold_presence,
4568 );
4569 Ok((admission, bound))
4570 });
4571 match admitted {
4572 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4573 Err(refusal) => {
4574 return Ok(vec![self.route_open_refusal_frame(
4575 ctx,
4576 &frame,
4577 &target_module_id,
4578 refusal.code,
4579 refusal.code,
4580 refusal.message,
4581 )?]);
4582 }
4583 }
4584 }
4585 };
4586
4587 timings.enter(RouteOpenPhase::ProjectRoot);
4609 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4610 Ok(project_root) => project_root,
4611 Err(err) => {
4612 return Ok(vec![control_error_frame(
4613 &frame,
4614 "invalid_project_root",
4615 err.to_string(),
4616 )?])
4617 }
4618 };
4619 timings.enter(RouteOpenPhase::Admission);
4620 identity.project_root = project_root.as_path().to_path_buf();
4621
4622 let _concurrency_guard = match self
4633 .route_bind_concurrency
4634 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4635 {
4636 Ok(guard) => guard,
4637 Err(in_flight) => {
4638 return Ok(vec![self.route_open_target_capacity_refusal(
4639 ctx,
4640 &frame,
4641 &target_module_id,
4642 in_flight,
4643 )?]);
4644 }
4645 };
4646
4647 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4650 RouteBindAdmission::Admitted { guard, probe } => {
4651 if probe {
4652 info!(
4653 module_id = %target_module_id,
4654 connection_id = ctx.connection_id.get(),
4655 "route.bind breaker half-open: admitting one probe"
4656 );
4657 }
4658 guard
4659 }
4660 RouteBindAdmission::Refused {
4661 consecutive_timeouts,
4662 retry_in,
4663 probe_in_flight,
4664 } => {
4665 return Ok(vec![self.route_open_breaker_refusal_frame(
4666 ctx,
4667 &frame,
4668 &target_module_id,
4669 consecutive_timeouts,
4670 retry_in,
4671 probe_in_flight,
4672 )?]);
4673 }
4674 };
4675
4676 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4681 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4682 timings.enter(RouteOpenPhase::EgressReservation);
4683 let pending = match self
4684 .forwarding
4685 .begin_route_bind_relay_for(
4686 ctx.connection_id,
4687 ctx.egress.clone(),
4688 response_version(&frame),
4689 frame.header.corr,
4690 &target_module_id,
4691 principal.clone(),
4692 bound_scope,
4693 Some(project_root),
4694 relay_deadline,
4695 )
4696 .await
4697 {
4698 Ok(pending) => pending,
4699 Err(err) => {
4700 return Ok(vec![self.route_open_refusal_frame(
4701 ctx,
4702 &frame,
4703 &target_module_id,
4704 "relay_reservation_failed",
4705 forwarding_error_code(&err),
4706 err.to_string(),
4707 )?])
4708 }
4709 };
4710 timings.enter(RouteOpenPhase::Admission);
4711 let crate::forwarding::PendingRouteBindRelay {
4712 endpoint,
4713 module_sink,
4714 negotiated_ver,
4715 client_channel,
4716 client_epoch,
4717 module_channel,
4718 module_epoch,
4719 corr: relay_corr,
4720 receiver,
4721 } = pending;
4722 let mut reservation =
4723 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4724
4725 let relay_registration = self
4730 .registry
4731 .get_module_by_connection(endpoint.connection_id)
4732 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4733 let relay_capabilities = relay_registration
4734 .as_ref()
4735 .and_then(|registration| registration.manifest.capabilities.as_ref());
4736 if let Some(selector) = &scope {
4737 if let Err(refusal) = self
4740 .scopes
4741 .read()
4742 .unwrap_or_else(|poisoned| poisoned.into_inner())
4743 .check_bind_proof(&principal, selector)
4744 {
4745 reservation.release_and_disarm();
4746 return Ok(vec![self.route_open_refusal_frame(
4747 ctx,
4748 &frame,
4749 &target_module_id,
4750 refusal.code,
4751 refusal.code,
4752 refusal.message,
4753 )?]);
4754 }
4755 }
4756 if let Some(stamp) = scope_stamp
4757 .as_ref()
4758 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4759 {
4760 if let Err(refusal) = crate::scopes::check_target_flow_support(
4761 stamp,
4762 &target_module_id,
4763 relay_capabilities,
4764 )
4765 .and_then(|()| {
4766 crate::scopes::check_target_agent_run_support(
4767 stamp,
4768 &target_module_id,
4769 relay_capabilities,
4770 )
4771 }) {
4772 reservation.release_and_disarm();
4773 return Ok(vec![self.route_open_refusal_frame(
4774 ctx,
4775 &frame,
4776 &target_module_id,
4777 refusal.code,
4778 refusal.code,
4779 refusal.message,
4780 )?]);
4781 }
4782 }
4783
4784 if !relay_capabilities.is_some_and(|caps| {
4787 caps.provides
4788 .iter()
4789 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
4790 }) {
4791 if let Some(stamp) = scope_stamp.as_mut() {
4792 stamp.attributes.on_behalf_of_agent = None;
4793 }
4794 }
4795 let call_mode = if relay_capabilities
4796 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4797 {
4798 Some(derive_call_mode(scope_stamp.as_ref()))
4799 } else {
4800 if let Some(stamp) = scope_stamp.as_mut() {
4801 stamp.expires_at_ms = None;
4802 }
4803 None
4804 };
4805
4806 debug!(
4807 connection_id = ctx.connection_id.get(),
4808 client_channel,
4809 client_epoch,
4810 module_channel,
4811 module_epoch,
4812 "reserved route handle pair"
4813 );
4814 let principal_label = principal_label(&principal);
4817 let relay = ModuleControlRequest::RouteBind {
4818 route_channel: module_channel,
4819 epoch: module_epoch,
4820 target,
4821 identity,
4822 principal: Some(principal),
4823 consumer_capabilities,
4824 role_versions,
4825 admission_facts,
4826 scope: scope_stamp,
4827 call_mode,
4828 };
4829 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4830 RouterError::backend(
4831 0,
4832 frame.header.corr,
4833 format!("failed to encode route.bind request: {err}"),
4834 )
4835 })?;
4836 let relay_frame = Frame::build_with_version(
4837 negotiated_ver,
4838 FrameType::Request,
4839 control_flags(),
4840 0,
4841 0,
4842 relay_corr,
4843 relay_body,
4844 )
4845 .map_err(RouterError::FrameBuild)?;
4846
4847 timings.enter(RouteOpenPhase::BindSend);
4848 let sent = module_sink.send(relay_frame).await;
4849 timings.enter(RouteOpenPhase::Admission);
4850 if let Err(err) = sent {
4851 reservation.release_and_disarm();
4852 return Ok(vec![self.route_open_refusal_frame(
4853 ctx,
4854 &frame,
4855 &target_module_id,
4856 "relay_send_failed",
4857 "target_unavailable",
4858 err.to_string(),
4859 )?]);
4860 }
4861
4862 #[cfg(test)]
4863 if let Some(hold) = &self.route_bind_relay_sent_hold {
4864 hold.wait().await;
4868 hold.wait().await;
4869 }
4870
4871 if self
4876 .forwarding
4877 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4878 .map_err(RouterError::Forwarding)?
4879 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4880 {
4881 self.send_abandoned_route_bind_goodbye(
4882 &module_sink,
4883 negotiated_ver,
4884 module_channel,
4885 module_epoch,
4886 );
4887 }
4888
4889 timings.enter(RouteOpenPhase::BindAckWait);
4890 let outcome = timeout_at(relay_deadline, receiver).await;
4891 timings.enter(RouteOpenPhase::Admission);
4892 match outcome {
4893 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4894 reservation.disarm();
4895 if breaker.record_accepted() {
4896 info!(
4897 module_id = %target_module_id,
4898 "route.bind breaker closed: the probe was accepted"
4899 );
4900 }
4901 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4902 Ok(Vec::new())
4903 }
4904 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4905 reservation.release_and_disarm();
4906 breaker.record_inconclusive();
4910 let scope_code = match body.code.as_str() {
4914 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4915 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4916 _ => None,
4917 };
4918 if let Some(code) = scope_code {
4919 self.observe_route_open_refusal(
4920 ctx,
4921 &target_module_id,
4922 "scope_changed_before_commit",
4923 code,
4924 );
4925 return Ok(vec![control_error_body_frame(&frame, body)?]);
4926 }
4927 self.counters
4928 .increment_route_open_refused("module_rejected");
4929 info!(
4930 target: "control",
4931 code = "module_rejected",
4932 module_code = ?body.code,
4933 module_id = ?target_module_id,
4934 connection_id = ctx.connection_id.get(),
4935 "route.open refused"
4936 );
4937 Ok(vec![control_error_body_frame(&frame, body)?])
4938 }
4939 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4940 reservation.release_and_disarm();
4941 breaker.record_inconclusive();
4942 tracing::warn!(
4949 module_id = %target_module_id,
4950 "route.bind relay abandoned: {message}"
4951 );
4952 Ok(vec![self.route_open_refusal_frame(
4953 ctx,
4954 &frame,
4955 &target_module_id,
4956 "relay_abandoned",
4957 "target_unavailable",
4958 message,
4959 )?])
4960 }
4961 Ok(Err(_)) => {
4962 reservation.release_and_disarm();
4963 breaker.record_inconclusive();
4964 Ok(vec![self.route_open_refusal_frame(
4965 ctx,
4966 &frame,
4967 &target_module_id,
4968 "relay_waiter_canceled",
4969 "target_unavailable",
4970 "route.bind relay waiter was canceled before the module responded",
4971 )?])
4972 }
4973 Err(_) => {
4974 reservation.release_and_disarm();
4975 if let Some(opened) = breaker.record_timeout(
4979 self.route_bind_breaker_threshold,
4980 self.route_bind_breaker_cooldown,
4981 ) {
4982 warn!(
4983 module_id = %target_module_id,
4984 consecutive_timeouts = opened.consecutive_timeouts,
4985 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4986 reopened_after_probe = opened.reopened_after_probe,
4987 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4988 );
4989 }
4990 tracing::warn!(
4998 module_id = %target_module_id,
4999 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5000 "route.bind relay timed out: module did not ack within budget"
5001 );
5002 Ok(vec![self.route_open_refusal_frame(
5003 ctx,
5004 &frame,
5005 &target_module_id,
5006 "relay_timed_out",
5007 "module_timeout",
5008 format!(
5009 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5010 route_bind_relay_timeout
5011 ),
5012 )?])
5013 }
5014 }
5015 }
5016
5017 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5018 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5019 snapshot: self.supervisor.spawn_snapshot(),
5020 };
5021 Ok(vec![control_response_body_frame(
5022 &frame,
5023 &response,
5024 "ClientControlResponse::SupervisorSpawnSnapshot",
5025 )?])
5026 }
5027
5028 fn handle_supervisor_spawn_subscribe(
5029 &self,
5030 ctx: &RouteCtx,
5031 frame: Frame,
5032 since: Option<SpawnCursor>,
5033 ) -> Result<Vec<Frame>, RouterError> {
5034 match self.supervisor.subscribe_spawns(
5035 ctx.connection_id,
5036 frame.header.corr,
5037 response_version(&frame),
5038 since,
5039 ctx.egress.clone(),
5040 ) {
5041 Ok(()) => Ok(Vec::new()),
5042 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5043 Ok(vec![control_error_body_frame(
5044 &frame,
5045 ErrorBody {
5046 code: "spawn_cursor_incarnation_mismatch".to_string(),
5047 message: "spawn cursor belongs to a different daemon incarnation"
5048 .to_string(),
5049 detail: Some(serde_json::json!({
5050 "current_daemon_incarnation": current
5051 })),
5052 },
5053 )?])
5054 }
5055 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5056 &frame,
5057 ErrorBody {
5058 code: "spawn_cursor_too_old".to_string(),
5059 message: "spawn cursor predates the retained event ring".to_string(),
5060 detail: Some(serde_json::json!({
5061 "oldest_retained_cursor": oldest
5062 })),
5063 },
5064 )?]),
5065 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5066 0,
5067 frame.header.corr,
5068 format!("failed to open supervisor spawn subscription: {error}"),
5069 )),
5070 }
5071 }
5072
5073 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5074 let generation = self
5075 .registry
5076 .generation()
5077 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5078 let mut modules = Vec::new();
5079 for module in self.supervisor.list() {
5080 let status = module.status_for_control("list").map_err(|err| {
5081 RouterError::backend(
5082 0,
5083 frame.header.corr,
5084 format!("failed to read supervisor status: {err}"),
5085 )
5086 })?;
5087 let (configured, _) = module.configuration().map_err(|err| {
5088 RouterError::backend(
5089 0,
5090 frame.header.corr,
5091 format!("failed to read module configuration: {err}"),
5092 )
5093 })?;
5094 let image = module.running_image_agreement().await;
5096 let resources = Some(module.child_resource_usage());
5099 let mut verdict =
5100 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5101 verdict.spec = Some(launch_spec_agreement(
5102 &configured,
5103 status.process_alive,
5104 status.launch_spec_digest.as_ref(),
5105 ));
5106 let pending_reload = Some(verdict);
5107 modules.push(
5110 SupervisorEntry::new(
5111 status.module_id,
5112 status.state.to_string(),
5113 status.enabled,
5114 status.live,
5115 status.health.status,
5116 )
5117 .with_launch_nonce_env(Some(!cfg!(unix)))
5118 .with_launch_nonce_source(status.launch_nonce_source)
5119 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5120 .with_protocol(status.protocol)
5121 .with_pending_reload(pending_reload)
5122 .with_last_probe_ms(status.health.last_probe_ms)
5123 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5124 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5125 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5126 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5127 .with_restart_count(Some(status.restart_count))
5128 .with_max_restarts(Some(status.max_restarts))
5129 .with_lifetime_restarts(Some(status.lifetime_restarts))
5130 .with_spawn_generation(Some(status.spawn_generation))
5131 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5132 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5133 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5134 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5135 .with_resources(resources),
5136 );
5137 }
5138 let response = ClientControlResponse::SupervisorList {
5139 generation,
5140 modules,
5141 };
5142 Ok(vec![control_response_body_frame(
5143 &frame,
5144 &response,
5145 "ClientControlResponse::SupervisorList",
5146 )?])
5147 }
5148
5149 fn handle_supervisor_stderr_tail(
5150 &self,
5151 frame: Frame,
5152 module_id: String,
5153 max_lines: Option<u32>,
5154 max_bytes: Option<u32>,
5155 ) -> Result<Vec<Frame>, RouterError> {
5156 let Some(module) = self.supervisor.get(&module_id) else {
5157 return Ok(vec![control_error_frame(
5158 &frame,
5159 "unknown_module",
5160 format!("module_id '{module_id}' is not supervised"),
5161 )?]);
5162 };
5163
5164 let snapshot = module.stderr_tail(
5165 max_lines.map(|value| value as usize),
5166 max_bytes.map(|value| value as usize),
5167 );
5168
5169 let response = ClientControlResponse::SupervisorStderrTail {
5170 module_id,
5171 tail: StderrTail {
5172 capture: match snapshot.capture {
5173 CaptureState::Captured => StderrCaptureState::Captured,
5174 CaptureState::Incomplete { reason } => {
5175 StderrCaptureState::Incomplete { reason }
5176 }
5177 CaptureState::NotCaptured { reason } => {
5178 StderrCaptureState::NotCaptured { reason }
5179 }
5180 },
5181 entries: snapshot
5182 .entries
5183 .into_iter()
5184 .map(|entry| match entry {
5185 TailEntry::Line {
5186 text,
5187 truncated,
5188 at_ms,
5189 } => StderrTailEntry::Line {
5190 text,
5191 truncated,
5192 at_ms,
5193 },
5194 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5195 })
5196 .collect(),
5197 dropped_lines: snapshot.dropped_lines,
5198 },
5199 };
5200 Ok(vec![control_response_body_frame(
5201 &frame,
5202 &response,
5203 "ClientControlResponse::SupervisorStderrTail",
5204 )?])
5205 }
5206
5207 async fn handle_supervisor_terminals(
5208 &self,
5209 frame: Frame,
5210 module_id: String,
5211 ) -> Result<Vec<Frame>, RouterError> {
5212 let Some(module) = self.supervisor.get(&module_id) else {
5213 return Ok(vec![control_error_frame(
5214 &frame,
5215 "unknown_module",
5216 format!("module_id '{module_id}' is not supervised"),
5217 )?]);
5218 };
5219
5220 let terminals = module
5223 .read_durable_terminal_history()
5224 .await
5225 .map_err(|error| {
5226 RouterError::backend(
5227 0,
5228 frame.header.corr,
5229 format!("failed to read terminal history: {error}"),
5230 )
5231 })?;
5232 let response = ClientControlResponse::SupervisorTerminals {
5233 module_id,
5234 terminals,
5235 };
5236 Ok(vec![control_response_body_frame(
5237 &frame,
5238 &response,
5239 "ClientControlResponse::SupervisorTerminals",
5240 )?])
5241 }
5242
5243 fn handle_supervisor_routes(
5244 &self,
5245 frame: Frame,
5246 module_id: Option<String>,
5247 ) -> Result<Vec<Frame>, RouterError> {
5248 let modules = self
5249 .forwarding
5250 .route_census(module_id.as_deref())
5251 .map_err(RouterError::Forwarding)?
5252 .into_iter()
5253 .map(|(module_id, routes)| SupervisorRouteModule {
5254 module_id,
5255 routes: routes
5256 .into_iter()
5257 .map(|route| SupervisorRoute {
5258 consumer: match route.principal {
5259 Principal::Reserved { module_id } => {
5260 SupervisorRouteConsumer::Reserved { module_id }
5261 }
5262 Principal::Direct | Principal::Unverified => {
5263 SupervisorRouteConsumer::Direct {
5264 connection_id: route.goodbye_target.connection_id.get(),
5265 }
5266 }
5267 },
5268 age_ms: Instant::now()
5269 .saturating_duration_since(route.bound_at)
5270 .as_millis()
5271 .try_into()
5272 .unwrap_or(u64::MAX),
5273 draining: route.draining,
5274 drain_reason: route.drain_reason,
5275 })
5276 .collect(),
5277 })
5278 .collect();
5279 let response = ClientControlResponse::SupervisorRoutes { modules };
5280 Ok(vec![control_response_body_frame(
5281 &frame,
5282 &response,
5283 "ClientControlResponse::SupervisorRoutes",
5284 )?])
5285 }
5286
5287 async fn handle_supervisor_provenance(
5288 &self,
5289 frame: Frame,
5290 module_id: Option<String>,
5291 ) -> Result<Vec<Frame>, RouterError> {
5292 let mut selected = if let Some(module_id) = module_id {
5293 let Some(module) = self.supervisor.get(&module_id) else {
5294 return Ok(vec![control_error_frame(
5295 &frame,
5296 "unknown_module",
5297 format!("module_id '{module_id}' is not supervised"),
5298 )?]);
5299 };
5300 vec![module]
5301 } else {
5302 self.supervisor.list()
5303 };
5304
5305 let mut modules = Vec::with_capacity(selected.len());
5306 for module in selected.drain(..) {
5307 let (status, observed_image) = module
5308 .status_and_running_image_agreement()
5309 .await
5310 .map_err(|err| {
5311 RouterError::backend(
5312 0,
5313 frame.header.corr,
5314 format!("failed to read supervisor status: {err}"),
5315 )
5316 })?;
5317 let module_declared = self
5318 .registry
5319 .get_module(&status.module_id)
5320 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5321 .and_then(|registration| registration.manifest.provenance)
5322 .map(|build| ModuleDeclaredProvenance::Reported { build })
5323 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5324 #[cfg(test)]
5325 let running_image = match &self.provenance_probe_override {
5326 Some(result) => result.clone(),
5327 None => observed_image,
5328 };
5329 #[cfg(not(test))]
5330 let running_image = observed_image;
5331 modules.push(SupervisorModuleProvenance {
5332 module_id: status.module_id,
5333 module_declared,
5334 daemon_observed: SupervisorObservedProcess {
5335 launch_nonce_source: status.launch_nonce_source,
5336 pid: status.pid,
5337 spawned_at_ms: status.spawned_at_ms,
5338 spawned_from: status.spawned_from,
5339 running_image,
5340 },
5341 });
5342 }
5343 let daemon = SupervisorDaemonProvenance {
5344 daemon_build: self.daemon_provenance.build.clone(),
5345 daemon_observed: DaemonObservedProcess {
5346 pid: self.daemon_provenance.pid,
5347 started_at_ms: self
5348 .daemon_provenance
5349 .start_clock
5350 .map(|clock| clock.started_at_ms())
5351 .or(self.daemon_provenance.started_at_ms),
5352 running_image: self
5353 .daemon_provenance
5354 .probe
5355 .observe(
5356 self.daemon_provenance.pid,
5357 self.daemon_provenance.executable_path.as_deref(),
5358 self.daemon_provenance.executable_identity.clone(),
5359 self.daemon_provenance.process_start_time,
5360 )
5361 .await,
5362 },
5363 };
5364 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5365 Ok(vec![control_response_body_frame(
5366 &frame,
5367 &response,
5368 "ClientControlResponse::SupervisorProvenance",
5369 )?])
5370 }
5371
5372 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5373 self.refresh_capability_requirements();
5374 let generation = self
5375 .registry
5376 .generation()
5377 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5378 let modules = self
5379 .supervisor
5380 .list()
5381 .into_iter()
5382 .map(|module| {
5383 let status = module.status_for_control("health").map_err(|err| {
5384 RouterError::backend(
5385 0,
5386 frame.header.corr,
5387 format!("failed to read supervisor health: {err}"),
5388 )
5389 })?;
5390 let module_id = status.module_id;
5391 let capability_detail = self
5392 .capability_evaluator
5393 .required_problem_detail(&module_id);
5394 Ok(SupervisorHealthEntry {
5395 module_id,
5396 status: status.health.status,
5397 detail: append_capability_problem_detail(
5398 status.health.detail,
5399 capability_detail,
5400 ),
5401 metrics: status.health.metrics,
5402 consecutive_failures: status.health.consecutive_failures,
5403 late_answer_count: status.health.late_answer_count,
5404 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5405 last_action: status.health.last_action,
5406 last_action_ms: status.health.last_action_ms,
5407 last_probe_ms: status.health.last_probe_ms,
5408 })
5409 })
5410 .collect::<Result<Vec<_>, RouterError>>()?;
5411 let response = ClientControlResponse::SupervisorHealth {
5412 generation,
5413 modules,
5414 };
5415 Ok(vec![control_response_body_frame(
5416 &frame,
5417 &response,
5418 "ClientControlResponse::SupervisorHealth",
5419 )?])
5420 }
5421
5422 async fn handle_supervisor_restart(
5423 &self,
5424 frame: Frame,
5425 module_id: String,
5426 drain_timeout_ms: Option<u64>,
5427 requester: Principal,
5428 ) -> Result<Vec<Frame>, RouterError> {
5429 let operation_lock = self.supervisor.operation_lock();
5430 let _operation_guard = operation_lock.lock().await;
5431 let Some(module) = self.supervisor.get(&module_id) else {
5432 return Ok(vec![control_error_frame(
5433 &frame,
5434 "unknown_module",
5435 format!("module_id '{module_id}' is not supervised"),
5436 )?]);
5437 };
5438
5439 self.route_outages.mark_operator_action(&module_id);
5440 if let Err(err) = module
5441 .restart_requested(drain_timeout_ms, Some(requester))
5442 .await
5443 {
5444 self.route_outages
5445 .operator_action_ended_unrefused(&module_id);
5446 let (code, message) = match err {
5447 crate::supervise::SuperviseError::Disabled { .. } => {
5448 ("module_disabled", err.to_string())
5449 }
5450 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5451 ("swap_in_progress", err.to_string())
5452 }
5453 _ => (
5454 "target_unavailable",
5455 format!("failed to restart module_id '{module_id}': {err}"),
5456 ),
5457 };
5458 return Ok(vec![control_error_frame(&frame, code, message)?]);
5459 }
5460
5461 let response = ClientControlResponse::SupervisorAck {
5462 module_id,
5463 applied: true,
5464 };
5465 Ok(vec![control_response_body_frame(
5466 &frame,
5467 &response,
5468 "ClientControlResponse::SupervisorAck",
5469 )?])
5470 }
5471
5472 async fn handle_supervisor_swap(
5476 &self,
5477 frame: Frame,
5478 module_id: String,
5479 ready_timeout_ms: Option<u64>,
5480 requester: Principal,
5481 ) -> Result<Vec<Frame>, RouterError> {
5482 let module = {
5489 let operation_lock = self.supervisor.operation_lock();
5490 let _operation_guard = operation_lock.lock().await;
5491 self.supervisor.get(&module_id)
5492 };
5493 let Some(module) = module else {
5494 return Ok(vec![control_error_frame(
5495 &frame,
5496 "unknown_module",
5497 format!("module_id '{module_id}' is not supervised"),
5498 )?]);
5499 };
5500
5501 self.route_outages.mark_operator_action(&module_id);
5502 if let Err(err) = module
5503 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5504 .await
5505 {
5506 self.route_outages
5507 .operator_action_ended_unrefused(&module_id);
5508 use crate::supervise::SuperviseError;
5509 let message = err.to_string();
5510 let error = match err {
5511 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5512 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5513 code: "swap_refused".to_string(),
5514 message,
5515 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5516 },
5517 SuperviseError::SwapFailed {
5518 arm,
5519 candidate_exit,
5520 ..
5521 } => ErrorBody {
5522 code: "swap_failed".to_string(),
5523 message,
5524 detail: Some(serde_json::json!({
5525 "arm": arm.as_str(),
5526 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5527 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5528 })),
5529 },
5530 _ => ErrorBody::new(
5531 "target_unavailable",
5532 format!("failed to swap module_id '{module_id}': {message}"),
5533 ),
5534 };
5535 return Ok(vec![control_error_body_frame(&frame, error)?]);
5536 }
5537 self.route_outages
5541 .operator_action_ended_unrefused(&module_id);
5542
5543 let response = ClientControlResponse::SupervisorAck {
5544 module_id,
5545 applied: true,
5546 };
5547 Ok(vec![control_response_body_frame(
5548 &frame,
5549 &response,
5550 "ClientControlResponse::SupervisorAck",
5551 )?])
5552 }
5553
5554 async fn handle_supervisor_reload(
5555 &self,
5556 frame: Frame,
5557 module_id: String,
5558 requester: Principal,
5559 ) -> Result<Vec<Frame>, RouterError> {
5560 let operation_lock = self.supervisor.operation_lock();
5561 let _operation_guard = operation_lock.lock().await;
5562 let Some(module) = self.supervisor.get(&module_id) else {
5563 return Ok(vec![control_error_frame(
5564 &frame,
5565 "unknown_module",
5566 format!("module_id '{module_id}' is not supervised"),
5567 )?]);
5568 };
5569
5570 self.route_outages.mark_operator_action(&module_id);
5571 if let Err(err) = module.reload_requested(Some(requester)).await {
5572 self.route_outages
5573 .operator_action_ended_unrefused(&module_id);
5574 let (code, message) = match err {
5575 crate::supervise::SuperviseError::Disabled { .. } => {
5576 ("module_disabled", err.to_string())
5577 }
5578 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5579 ("swap_in_progress", err.to_string())
5580 }
5581 _ => (
5582 "reload_failed",
5583 format!("failed to reload module_id '{module_id}': {err}"),
5584 ),
5585 };
5586 return Ok(vec![control_error_frame(&frame, code, message)?]);
5587 }
5588
5589 let response = ClientControlResponse::SupervisorAck {
5590 module_id,
5591 applied: true,
5592 };
5593 Ok(vec![control_response_body_frame(
5594 &frame,
5595 &response,
5596 "ClientControlResponse::SupervisorAck",
5597 )?])
5598 }
5599
5600 async fn handle_supervisor_rescan(
5601 &self,
5602 frame: Frame,
5603 preview: bool,
5604 requester: Principal,
5605 ) -> Result<Vec<Frame>, RouterError> {
5606 let Some(context) = self.rescan.clone() else {
5607 return Ok(vec![control_error_frame(
5608 &frame,
5609 "rescan_unavailable",
5610 "the daemon was not started with a reloadable config path".to_string(),
5611 )?]);
5612 };
5613
5614 let operation_lock = self.supervisor.operation_lock();
5615 let _operation_guard = operation_lock.lock().await;
5616 let loaded = match crate::daemon_config::load_with_activity(&context.config_path) {
5617 Ok(config) => config,
5618 Err(err) => {
5619 return Ok(vec![control_error_frame(
5620 &frame,
5621 "invalid_daemon_config",
5622 format!("supervisor rescan rejected daemon config: {err}"),
5623 )?])
5624 }
5625 };
5626 let Some((config, activity_config)) = loaded else {
5634 return Ok(vec![control_error_frame(
5635 &frame,
5636 "invalid_daemon_config",
5637 format!(
5638 "daemon config not found at {}; refusing to rescan (an absent config would \
5639 retire every supervised module)",
5640 context.config_path.display()
5641 ),
5642 )?]);
5643 };
5644 let (
5645 configured_port,
5646 storage_config,
5647 admission_facts_carrier_module_id,
5648 admission_facts_targets,
5649 scope_authority_owners,
5650 privileged_operator_prompt,
5651 modules,
5652 reserved_capabilities,
5653 ) = (
5654 config.port,
5655 config.storage,
5656 config.admission_facts_carrier_module_id,
5657 config.admission_facts_targets,
5658 config.scope_authority_owners,
5659 config.privileged_operator_prompt,
5660 config.modules,
5661 config.reserved_capabilities,
5662 );
5663
5664 let mut restart_required = Vec::new();
5673 for section in RestartRequiredSection::ALL {
5674 let changed = match section {
5675 RestartRequiredSection::Port => configured_port != context.configured_port,
5676 RestartRequiredSection::Storage => storage_config != context.storage_config,
5677 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5678 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5679 }
5680 RestartRequiredSection::AdmissionFactsTargets => {
5681 admission_facts_targets != context.admission_facts_targets
5682 }
5683 RestartRequiredSection::ScopeAuthorityOwners => {
5684 scope_authority_owners != context.scope_authority_owners
5685 }
5686 RestartRequiredSection::PrivilegedOperatorPrompt => {
5687 privileged_operator_prompt != context.privileged_operator_prompt
5688 }
5689 };
5690 if changed {
5691 restart_required.push(section.label().to_string());
5692 }
5693 }
5694 if !restart_required.is_empty() {
5695 warn!(
5696 config_path = %context.config_path.display(),
5697 sections = %restart_required.join(", "),
5698 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5699 );
5700 }
5701
5702 for configured in &modules {
5703 if let Err(err) = validate_spec(&configured.module_spec()) {
5704 return Ok(vec![control_error_frame(
5705 &frame,
5706 "invalid_daemon_config",
5707 format!("supervisor rescan rejected daemon config: {err}"),
5708 )?]);
5709 }
5710 }
5711
5712 let configured_capabilities = modules
5713 .iter()
5714 .map(|module| (module.module_id.clone(), module.enabled))
5715 .collect::<Vec<_>>();
5716 let preview_capability_warnings = if preview {
5717 let (_, registrations) = self.runtime_capability_snapshot()?;
5718 let current_modules = self
5719 .supervisor
5720 .list()
5721 .into_iter()
5722 .map(|module| module.module_id().to_string())
5723 .collect::<BTreeSet<_>>();
5724 let resulting_modules = configured_capabilities.clone();
5725 let removed = current_modules
5726 .into_iter()
5727 .filter(|module_id| {
5728 !resulting_modules
5729 .iter()
5730 .any(|(configured_id, _)| configured_id == module_id)
5731 })
5732 .collect::<Vec<_>>();
5733 self.capability_evaluator.preview_removal_warnings(
5734 resulting_modules,
5735 &removed,
5736 ®istrations,
5737 )
5738 } else {
5739 Vec::new()
5740 };
5741 let result = match self
5742 .reconcile_supervised_modules(&context.supervisor, modules, preview, requester.clone())
5743 .await
5744 {
5745 Ok(result) => result,
5746 Err(message) => {
5747 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5748 }
5749 };
5750 if !preview {
5751 if let Some(ledger) = &self.activity {
5752 ledger.module_rescanned(&result.added, &result.removed, requester);
5753 }
5754 if let Err(code) = self.configure_activity(activity_config).await {
5755 warn!(
5756 code,
5757 "activity retention could not be applied during rescan"
5758 );
5759 }
5760 self.capability_evaluator
5761 .configure(configured_capabilities, reserved_capabilities);
5762 self.capability_evaluator.wake_deadline_loop();
5763 self.refresh_capability_requirements();
5764 }
5765 let mut result = result;
5766 result.restart_required = restart_required;
5767 result.capability_warnings = preview_capability_warnings;
5768 let response = ClientControlResponse::SupervisorRescan { result };
5769 Ok(vec![control_response_body_frame(
5770 &frame,
5771 &response,
5772 "ClientControlResponse::SupervisorRescan",
5773 )?])
5774 }
5775
5776 async fn handle_supervisor_release_reserved(
5777 &self,
5778 frame: Frame,
5779 module_id: String,
5780 ) -> Result<Vec<Frame>, RouterError> {
5781 let Some(context) = self.rescan.clone() else {
5782 return Ok(vec![control_error_frame(
5783 &frame,
5784 "release_unavailable",
5785 "reserved-id release requires a daemon started with a reloadable config path",
5786 )?]);
5787 };
5788 let operation_lock = self.supervisor.operation_lock();
5789 let _operation_guard = operation_lock.lock().await;
5790 let loaded = match crate::daemon_config::load(&context.config_path) {
5791 Ok(Some(config)) => config,
5792 Ok(None) => {
5793 return Ok(vec![control_error_frame(
5794 &frame,
5795 "invalid_daemon_config",
5796 format!(
5797 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5798 context.config_path.display()
5799 ),
5800 )?])
5801 }
5802 Err(err) => {
5803 return Ok(vec![control_error_frame(
5804 &frame,
5805 "invalid_daemon_config",
5806 format!("unable to verify reserved-id release against daemon config: {err}"),
5807 )?])
5808 }
5809 };
5810 if loaded
5811 .modules
5812 .iter()
5813 .any(|configured| configured.module_id == module_id)
5814 {
5815 return Ok(vec![control_error_frame(
5816 &frame,
5817 "reserved_module_configured",
5818 format!(
5819 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5820 ),
5821 )?]);
5822 }
5823 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5824 return Ok(vec![control_error_frame(
5825 &frame,
5826 "reserved_gate_not_retained",
5827 format!(
5828 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5829 ),
5830 )?]);
5831 }
5832
5833 let response = ClientControlResponse::SupervisorAck {
5834 module_id,
5835 applied: true,
5836 };
5837 Ok(vec![control_response_body_frame(
5838 &frame,
5839 &response,
5840 "ClientControlResponse::SupervisorAck",
5841 )?])
5842 }
5843
5844 async fn reconcile_supervised_modules(
5853 &self,
5854 supervisor: &Supervisor,
5855 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5856 preview: bool,
5857 requester: Principal,
5858 ) -> Result<SupervisorRescanResult, String> {
5859 let mut current = BTreeMap::new();
5860 for module in self.supervisor.list() {
5861 let (spec, health) = module.configuration().map_err(|err| {
5862 format!(
5863 "failed to read configuration for module_id '{}': {err}",
5864 module.module_id()
5865 )
5866 })?;
5867 let enabled = module
5868 .status()
5869 .map_err(|err| {
5870 format!(
5871 "failed to read status for module_id '{}': {err}",
5872 module.module_id()
5873 )
5874 })?
5875 .enabled;
5876 current.insert(
5877 module.module_id().to_string(),
5878 (module, spec, health, enabled),
5879 );
5880 }
5881 let configured = configured_modules
5882 .into_iter()
5883 .map(|module| (module.module_id.clone(), module))
5884 .collect::<BTreeMap<_, _>>();
5885
5886 let added = configured
5887 .keys()
5888 .filter(|module_id| !current.contains_key(*module_id))
5889 .cloned()
5890 .collect::<Vec<_>>();
5891 let removed = current
5892 .keys()
5893 .filter(|module_id| !configured.contains_key(*module_id))
5894 .cloned()
5895 .collect::<Vec<_>>();
5896 let mut changed_pending_reload = Vec::new();
5897 let mut configuration_changes = BTreeSet::new();
5898 let mut enabled_changes = BTreeSet::new();
5899 let mut unchanged = 0_u32;
5900
5901 for (module_id, configured_module) in &configured {
5902 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5903 else {
5904 continue;
5905 };
5906 let launch_changed = *current_spec != configured_module.module_spec();
5910 let configuration_changed =
5911 launch_changed || *current_health != configured_module.health;
5912 let enabled_changed = *current_enabled != configured_module.enabled;
5913 if configuration_changed {
5914 configuration_changes.insert(module_id.clone());
5915 }
5916 if launch_changed {
5917 changed_pending_reload.push(module_id.clone());
5918 }
5919 if enabled_changed {
5920 enabled_changes.insert(module_id.clone());
5921 }
5922 if !configuration_changed && !enabled_changed {
5923 unchanged = unchanged.saturating_add(1);
5924 }
5925 }
5926
5927 if preview {
5933 return Ok(SupervisorRescanResult {
5934 added,
5935 removed,
5936 changed_pending_reload,
5937 enabled_changes: enabled_changes.iter().cloned().collect(),
5938 unchanged,
5939 preview: true,
5940 restart_required: Vec::new(),
5944 capability_warnings: Vec::new(),
5945 });
5946 }
5947
5948 for module_id in &removed {
5949 let module = ¤t
5950 .get(module_id)
5951 .expect("removed module came from current supervisor state")
5952 .0;
5953 module
5954 .retire_requested(Some(requester.clone()))
5955 .await
5956 .map_err(|err| {
5957 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5958 })?;
5959 self.supervisor.record_rescan_removal(module_id);
5993 self.supervisor.retire(module_id);
5994 self.route_outages.forget(module_id);
5995 }
5996
5997 for module_id in configured.keys() {
5998 let Some((module, _, _, _)) = current.get(module_id) else {
5999 continue;
6000 };
6001 let configured_module = configured
6002 .get(module_id)
6003 .expect("configured module id came from configured map");
6004 if configuration_changes.contains(module_id) {
6005 module
6006 .update_configuration(
6007 configured_module.module_spec(),
6008 configured_module.health.clone(),
6009 configured_module.drain_timeout_ms,
6010 )
6011 .await
6012 .map_err(|err| {
6013 format!(
6014 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6015 )
6016 })?;
6017 }
6018 if enabled_changes.contains(module_id) {
6019 self.route_outages.mark_operator_action(module_id);
6022 module
6023 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6024 .await
6025 .map_err(|err| {
6026 self.route_outages.operator_action_ended_unrefused(module_id);
6027 format!(
6028 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6029 configured_module.enabled
6030 )
6031 })?;
6032 }
6033 }
6034
6035 for module_id in &added {
6036 let configured_module = configured
6037 .get(module_id)
6038 .expect("added module id came from configured map");
6039 supervisor
6040 .supervise_configured_requested(
6041 configured_module.module_spec(),
6042 configured_module.enabled,
6043 configured_module.health.clone(),
6044 configured_module.drain_timeout_ms,
6045 configured_module.restart,
6046 Some(requester.clone()),
6047 )
6048 .map_err(|err| {
6049 format!("failed to add module_id '{module_id}' during rescan: {err}")
6050 })?;
6051 }
6052
6053 Ok(SupervisorRescanResult {
6054 added,
6055 removed,
6056 changed_pending_reload,
6057 enabled_changes: enabled_changes.iter().cloned().collect(),
6058 unchanged,
6059 preview: false,
6060 restart_required: Vec::new(),
6063 capability_warnings: Vec::new(),
6064 })
6065 }
6066
6067 async fn handle_supervisor_set_enabled(
6068 &self,
6069 frame: Frame,
6070 module_id: String,
6071 enabled: bool,
6072 requester: Principal,
6073 ) -> Result<Vec<Frame>, RouterError> {
6074 let operation_lock = self.supervisor.operation_lock();
6075 let _operation_guard = operation_lock.lock().await;
6076 let Some(module) = self.supervisor.get(&module_id) else {
6077 return Ok(vec![control_error_frame(
6078 &frame,
6079 "unknown_module",
6080 format!("module_id '{module_id}' is not supervised"),
6081 )?]);
6082 };
6083
6084 self.route_outages.mark_operator_action(&module_id);
6087 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6088 Ok(applied) => applied,
6089 Err(err) => {
6090 self.route_outages
6091 .operator_action_ended_unrefused(&module_id);
6092 return Ok(vec![control_error_frame(
6093 &frame,
6094 "target_unavailable",
6095 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6096 )?]);
6097 }
6098 };
6099 if !applied {
6100 self.route_outages
6103 .operator_action_ended_unrefused(&module_id);
6104 }
6105
6106 self.capability_evaluator.wake_deadline_loop();
6107 self.refresh_capability_requirements();
6108 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6109 Ok(vec![control_response_body_frame(
6110 &frame,
6111 &response,
6112 "ClientControlResponse::SupervisorAck",
6113 )?])
6114 }
6115
6116 async fn handle_supervisor_health_probe(
6117 &self,
6118 frame: Frame,
6119 module_id: String,
6120 ) -> Result<Vec<Frame>, RouterError> {
6121 self.refresh_capability_requirements();
6122 let Some(registration) = self
6123 .registry
6124 .get_module(&module_id)
6125 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6126 else {
6127 return Ok(vec![control_error_frame(
6128 &frame,
6129 "unknown_module",
6130 format!("module_id '{module_id}' is not registered"),
6131 )?]);
6132 };
6133
6134 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6147 {
6148 return Ok(vec![control_error_frame(
6149 &frame,
6150 "health_not_advertised",
6151 format!("module_id '{module_id}' did not advertise health.check"),
6152 )?]);
6153 }
6154
6155 let deadline = Instant::now() + self.health_probe_timeout;
6156 let pending = match self.forwarding.begin_module_control_rpc_for(
6157 &module_id,
6158 MODULE_CONTROL_OP_HEALTH_CHECK,
6159 deadline,
6160 ) {
6161 Ok(pending) => pending,
6162 Err(err) => {
6163 return Ok(vec![control_error_frame(
6164 &frame,
6165 forwarding_error_code(&err),
6166 err.to_string(),
6167 )?])
6168 }
6169 };
6170
6171 let PendingModuleControlRpc {
6172 endpoint,
6173 module_sink,
6174 negotiated_ver,
6175 corr: probe_corr,
6176 receiver,
6177 } = pending;
6178 let mut guard =
6179 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6180 let probe_body =
6181 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6182 RouterError::backend(
6183 0,
6184 frame.header.corr,
6185 format!("failed to encode health.check request: {err}"),
6186 )
6187 })?;
6188 let probe_frame = Frame::build_with_version(
6189 negotiated_ver,
6190 FrameType::Request,
6191 control_flags(),
6192 0,
6193 0,
6194 probe_corr,
6195 probe_body,
6196 )
6197 .map_err(RouterError::FrameBuild)?;
6198
6199 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6202 Ok(Ok(())) => {}
6203 Ok(Err(err)) => {
6204 return Ok(vec![control_error_frame(
6205 &frame,
6206 "target_unavailable",
6207 err.to_string(),
6208 )?]);
6209 }
6210 Err(_) => {
6211 return Ok(vec![control_error_frame(
6212 &frame,
6213 "module_timeout",
6214 format!(
6215 "module_id '{module_id}' did not answer health.check within {:?}",
6216 self.health_probe_timeout
6217 ),
6218 )?]);
6219 }
6220 }
6221
6222 match timeout_at(deadline, receiver).await {
6223 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6224 guard.disarm();
6225 let Some(report) = response.health_report() else {
6226 return Ok(vec![control_error_frame(
6227 &frame,
6228 "invalid_control_body",
6229 "health.check RPC returned a non-health response",
6230 )?]);
6231 };
6232 let HealthReport {
6237 status,
6238 detail,
6239 metrics,
6240 } = report;
6241 let capability_detail = self
6242 .capability_evaluator
6243 .required_problem_detail(&module_id);
6244 let response = ClientControlResponse::SupervisorHealthProbe {
6245 module_id,
6246 status,
6247 detail: append_capability_problem_detail(detail, capability_detail),
6248 metrics,
6249 };
6250 Ok(vec![control_response_body_frame(
6251 &frame,
6252 &response,
6253 "ClientControlResponse::SupervisorHealthProbe",
6254 )?])
6255 }
6256 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6257 guard.disarm();
6258 Ok(vec![control_error_body_frame(&frame, body)?])
6259 }
6260 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6261 guard.disarm();
6262 Ok(vec![control_error_frame(
6263 &frame,
6264 "target_unavailable",
6265 message,
6266 )?])
6267 }
6268 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6269 guard.disarm();
6270 Ok(vec![control_error_frame(
6271 &frame,
6272 "invalid_control_body",
6273 message,
6274 )?])
6275 }
6276 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6277 guard.disarm();
6278 Ok(vec![control_error_frame(
6279 &frame,
6280 "invalid_control_body",
6281 format!("expected module-control op '{expected}', got '{actual}'"),
6282 )?])
6283 }
6284 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6285 guard.disarm();
6286 Ok(vec![control_error_frame(
6287 &frame,
6288 "module_timeout",
6289 format!(
6290 "module_id '{module_id}' answered health.check after {:?}",
6291 self.health_probe_timeout
6292 ),
6293 )?])
6294 }
6295 Ok(Err(_)) => Ok(vec![control_error_frame(
6296 &frame,
6297 "target_unavailable",
6298 "health.check waiter was canceled before the module responded",
6299 )?]),
6300 Err(_) => Ok(vec![control_error_frame(
6301 &frame,
6302 "module_timeout",
6303 format!(
6304 "module_id '{module_id}' did not answer health.check within {:?}",
6305 self.health_probe_timeout
6306 ),
6307 )?]),
6308 }
6309 }
6310
6311 fn supervisor_status(
6312 &self,
6313 module_id: &str,
6314 corr: u64,
6315 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6316 self.supervisor
6317 .get(module_id)
6318 .map(|module| {
6319 let warming = module.is_warming_for_control("status").map_err(|err| {
6320 RouterError::backend(
6321 0,
6322 corr,
6323 format!(
6324 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6325 ),
6326 )
6327 })?;
6328 module.status_for_control("status").map_err(|err| {
6329 RouterError::backend(
6330 0,
6331 corr,
6332 format!(
6333 "failed to read supervisor status for module_id '{module_id}': {err}"
6334 ),
6335 )
6336 }).map(|status| (status, warming))
6337 })
6338 .transpose()
6339 }
6340
6341 fn guard_module_control_op(
6342 &self,
6343 frame: &Frame,
6344 module_id: &str,
6345 op: &str,
6346 ) -> Result<Option<Frame>, RouterError> {
6347 if self.module_grants_op(module_id, op, frame.header.corr)? {
6348 return Ok(None);
6349 }
6350
6351 Ok(Some(control_error_frame(
6352 frame,
6353 "op_not_allowed",
6354 format!("module_id '{module_id}' did not grant control op '{op}'"),
6355 )?))
6356 }
6357
6358 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6359 let Some(registration) = self
6360 .registry
6361 .get_module(module_id)
6362 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6363 else {
6364 return Ok(false);
6365 };
6366 Ok(module_registration_grants_op(®istration.control_ops, op))
6367 }
6368
6369 fn handle_status_update(
6370 &self,
6371 endpoint: ModuleEndpointId,
6372 frame: Frame,
6373 ) -> Result<Vec<Frame>, RouterError> {
6374 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6375 Ok(update) => update,
6376 Err(err) => {
6377 if is_known_module_push_op(&frame.body) {
6382 return Ok(vec![control_error_frame(
6383 &frame,
6384 "invalid_control_body",
6385 format!("malformed module control push body: {err}"),
6386 )?]);
6387 }
6388 return Ok(Vec::new());
6389 }
6390 };
6391
6392 match update {
6393 ModuleControlPush::RouteStatus {
6394 route_channel,
6395 route_epoch,
6396 status,
6397 } => {
6398 self.forwarding
6399 .cache_status(endpoint, route_channel, route_epoch, status)
6400 .map_err(RouterError::Forwarding)?;
6401 }
6402 }
6403 Ok(Vec::new())
6404 }
6405
6406 fn handle_route_poll(
6407 &self,
6408 ctx: &RouteCtx,
6409 frame: Frame,
6410 route_channel: u16,
6411 route_epoch: u32,
6412 kind: PollKind,
6413 ) -> Result<Vec<Frame>, RouterError> {
6414 let snapshot = self
6415 .forwarding
6416 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6417 .map_err(RouterError::Forwarding)?;
6418 let response = match (kind, snapshot) {
6419 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6420 ClientControlResponse::RoutePoll {
6421 route_channel,
6422 route_epoch,
6423 status,
6424 live: None,
6425 }
6426 }
6427 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6428 route_channel,
6429 route_epoch,
6430 status: None,
6431 live: None,
6432 },
6433 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6434 let live = self
6460 .process_liveness
6461 .as_ref()
6462 .and_then(|source| source.process_live(&module_id))
6463 .unwrap_or(true);
6464 ClientControlResponse::RoutePoll {
6465 route_channel,
6466 route_epoch,
6467 status: None,
6468 live: Some(live),
6469 }
6470 }
6471 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6472 route_channel,
6473 route_epoch,
6474 status: None,
6475 live: Some(false),
6476 },
6477 };
6478
6479 Ok(vec![control_response_body_frame(
6480 &frame,
6481 &response,
6482 "ClientControlResponse::RoutePoll",
6483 )?])
6484 }
6485
6486 pub(crate) fn observe_module_control_completion(
6487 &self,
6488 completion: ModuleControlRpcCompletion,
6489 ) -> bool {
6490 match completion {
6491 ModuleControlRpcCompletion::Unknown => false,
6492 ModuleControlRpcCompletion::Settled => true,
6493 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6494 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6495 info!(
6496 module_id = %module_id,
6497 latency_ms,
6498 "late health.check answer proves the module is alive"
6499 );
6500 match self
6501 .supervisor
6502 .record_late_health_answer(&module_id, latency_ms)
6503 {
6504 Ok(true) => {}
6505 Ok(false) => debug!(
6506 module_id = %module_id,
6507 latency_ms,
6508 "late health.check answer has no active supervisor snapshot"
6509 ),
6510 Err(err) => warn!(
6511 module_id = %module_id,
6512 latency_ms,
6513 error = %err,
6514 "failed to record late health.check answer"
6515 ),
6516 }
6517 true
6518 }
6519 }
6520 }
6521
6522 fn refuse_to_end_module_connection_for_a_client(
6543 &self,
6544 module_connection_id: ConnectionId,
6545 corr: u64,
6546 err: ForwardingError,
6547 ) -> Result<(), RouterError> {
6548 if let ForwardingError::ConnectionClosing { connection_id } = err {
6549 if connection_id != module_connection_id {
6550 warn!(
6551 module_connection_id = module_connection_id.get(),
6552 client_connection_id = connection_id.get(),
6553 corr,
6554 "dropping a route.bind response for a closing client; the module connection keeps serving"
6555 );
6556 return Ok(());
6557 }
6558 }
6559 Err(RouterError::Forwarding(err))
6560 }
6561
6562 fn handle_module_relay_response(
6563 &self,
6564 connection_id: ConnectionId,
6565 frame: Frame,
6566 ) -> Result<Vec<Frame>, RouterError> {
6567 let mut secondary_error = None;
6568 let outcome = match frame.header.ty {
6569 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6570 Ok(probe) if probe.op == "route.bind" => {
6571 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6572 Ok(ModuleControlResponse::RouteBindAck {}) => {
6573 RouteBindRelayOutcome::Accepted
6574 }
6575 Ok(other) => {
6576 let message =
6577 format!("route.bind response carried unexpected body: {other:?}");
6578 secondary_error = Some(control_error_frame(
6579 &frame,
6580 "invalid_control_body",
6581 message.clone(),
6582 )?);
6583 RouteBindRelayOutcome::ModuleGone(message)
6584 }
6585 Err(err) => {
6586 let message = format!("malformed route.bind response body: {err}");
6587 secondary_error = Some(control_error_frame(
6588 &frame,
6589 "invalid_control_body",
6590 message.clone(),
6591 )?);
6592 RouteBindRelayOutcome::ModuleGone(message)
6593 }
6594 }
6595 }
6596 Ok(probe) => {
6597 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6598 {
6599 Ok(response) => ModuleControlRpcOutcome::Response(response),
6600 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6601 "malformed {} response body: {err}",
6602 probe.op
6603 )),
6604 };
6605 let completion = self
6606 .forwarding
6607 .complete_module_control_rpc(
6608 connection_id,
6609 frame.header.corr,
6610 Some(&probe.op),
6611 outcome,
6612 )
6613 .map_err(RouterError::Forwarding)?;
6614 if !self.observe_module_control_completion(completion) {
6615 debug!(
6616 connection_id = connection_id.get(),
6617 corr = frame.header.corr,
6618 op = %probe.op,
6619 "dropping late or unknown module-control RPC response"
6620 );
6621 }
6622 return Ok(Vec::new());
6623 }
6624 Err(err) => {
6625 if let Some(expected_op) = self
6626 .forwarding
6627 .pending_module_control_op(connection_id, frame.header.corr)
6628 .map_err(RouterError::Forwarding)?
6629 {
6630 let completion = self
6631 .forwarding
6632 .complete_module_control_rpc(
6633 connection_id,
6634 frame.header.corr,
6635 None,
6636 ModuleControlRpcOutcome::MalformedResponse(format!(
6637 "malformed {expected_op} response body: {err}"
6638 )),
6639 )
6640 .map_err(RouterError::Forwarding)?;
6641 if !self.observe_module_control_completion(completion) {
6642 debug!(
6643 connection_id = connection_id.get(),
6644 corr = frame.header.corr,
6645 "dropping late malformed module-control RPC response"
6646 );
6647 }
6648 return Ok(Vec::new());
6649 }
6650 let message = format!("malformed route.bind response body: {err}");
6651 secondary_error = Some(control_error_frame(
6652 &frame,
6653 "invalid_control_body",
6654 message.clone(),
6655 )?);
6656 RouteBindRelayOutcome::ModuleGone(message)
6657 }
6658 },
6659 FrameType::Error => {
6660 if self
6661 .forwarding
6662 .pending_module_control_op(connection_id, frame.header.corr)
6663 .map_err(RouterError::Forwarding)?
6664 .is_some()
6665 {
6666 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6667 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6668 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6669 "malformed module-control ERROR body: {err}"
6670 )),
6671 };
6672 let completion = self
6673 .forwarding
6674 .complete_module_control_rpc(
6675 connection_id,
6676 frame.header.corr,
6677 None,
6678 outcome,
6679 )
6680 .map_err(RouterError::Forwarding)?;
6681 if !self.observe_module_control_completion(completion) {
6682 debug!(
6683 connection_id = connection_id.get(),
6684 corr = frame.header.corr,
6685 "dropping late or unknown module-control RPC error"
6686 );
6687 }
6688 return Ok(Vec::new());
6689 }
6690 match serde_json::from_slice::<ErrorBody>(&frame.body) {
6691 Ok(body) => RouteBindRelayOutcome::Rejected(body),
6692 Err(err) => {
6693 let message = format!("malformed route.bind ERROR body: {err}");
6694 secondary_error = Some(control_error_frame(
6695 &frame,
6696 "invalid_control_body",
6697 message.clone(),
6698 )?);
6699 RouteBindRelayOutcome::ModuleGone(message)
6700 }
6701 }
6702 }
6703 ty => {
6704 return Ok(vec![control_error_frame(
6705 &frame,
6706 "unsupported_control_frame",
6707 format!("unsupported module channel-0 frame {ty:?}"),
6708 )?])
6709 }
6710 };
6711
6712 let settled =
6713 self.forwarding
6714 .complete_pending_relay(connection_id, frame.header.corr, outcome);
6715 let completion = match settled {
6716 Ok(completion) => completion,
6717 Err(err) => {
6718 self.refuse_to_end_module_connection_for_a_client(
6719 connection_id,
6720 frame.header.corr,
6721 err,
6722 )?;
6723 return Ok(secondary_error.into_iter().collect());
6724 }
6725 };
6726 if let Some(target) = completion.abandoned.as_ref() {
6727 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6728 }
6729 if !completion.settled {
6730 debug!(
6731 connection_id = connection_id.get(),
6732 corr = frame.header.corr,
6733 frame_type = ?frame.header.ty,
6734 "dropping late or unknown route.bind relay response"
6735 );
6736 }
6737 Ok(secondary_error.into_iter().collect())
6738 }
6739
6740 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6741 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6742 self.cleanup_connection_with_end_reason(
6746 connection_id,
6747 RegistrationEndReason::ExplicitGoodbye,
6748 )
6749 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6750 Ok(Vec::new())
6751 }
6752}
6753
6754impl Default for ControlHandler {
6755 fn default() -> Self {
6756 Self::new(Arc::new(Registry::default()))
6757 }
6758}
6759
6760impl crate::supervise::SwapPromotionObserver for ControlHandler {
6761 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6762 self.apply_registration_capabilities(registration);
6763 }
6764}
6765
6766fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6767 CapabilityRequirementStatus {
6768 consumer: status.consumer,
6769 capability: status.capability,
6770 need: match status.need {
6771 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6772 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6773 },
6774 verdict: status.verdict.as_str().to_string(),
6775 episode_seq: status.episode_seq,
6776 config_satisfiable: status.config_satisfiable,
6777 runtime_available: status.runtime_available,
6778 detail: status.detail,
6779 }
6780}
6781
6782fn append_capability_problem_detail(
6783 detail: Option<String>,
6784 capability_detail: Option<String>,
6785) -> Option<String> {
6786 match (detail, capability_detail) {
6787 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6788 (Some(detail), None) => Some(detail),
6789 (None, Some(capability_detail)) => Some(capability_detail),
6790 (None, None) => None,
6791 }
6792}
6793
6794fn subc_ops() -> Vec<String> {
6795 SUBC_CONTROL_OPS
6796 .iter()
6797 .map(|op| (*op).to_string())
6798 .collect()
6799}
6800
6801fn module_subc_ops() -> Vec<String> {
6802 SUBC_CONTROL_OPS
6803 .iter()
6804 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6805 .map(|op| (*op).to_string())
6806 .collect()
6807}
6808
6809#[cfg(test)]
6810fn module_baseline_control_ops() -> Vec<String> {
6811 MODULE_BASELINE_CONTROL_OPS
6812 .iter()
6813 .map(|op| (*op).to_string())
6814 .collect()
6815}
6816
6817fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6818 let mut seen = HashSet::new();
6819 let mut effective = Vec::new();
6820 for op in MODULE_BASELINE_CONTROL_OPS {
6821 if seen.insert((*op).to_string()) {
6822 effective.push((*op).to_string());
6823 }
6824 }
6825 for op in declared.unwrap_or_default() {
6826 if seen.insert(op.clone()) {
6827 effective.push(op);
6828 }
6829 }
6830 effective
6831}
6832
6833fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6834 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6835}
6836
6837fn target_module_id(target: &RouteTarget) -> &str {
6838 match target {
6839 RouteTarget::ToolProvider { module_id }
6840 | RouteTarget::ManagementSurface { module_id }
6841 | RouteTarget::InternalService { module_id, .. } => module_id,
6842 }
6843}
6844
6845fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6846 roles.iter().any(|role| match (target, role) {
6847 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6848 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6849 (
6850 RouteTarget::InternalService { service_id, .. },
6851 ProviderRole::InternalService {
6852 service_id: provided,
6853 ..
6854 },
6855 ) => service_id == provided,
6856 _ => false,
6857 })
6858}
6859
6860fn is_routable_role(role: &ProviderRole) -> bool {
6861 matches!(
6862 role,
6863 ProviderRole::ToolProvider { .. }
6864 | ProviderRole::ManagementSurface { .. }
6865 | ProviderRole::InternalService { .. }
6866 )
6867}
6868
6869#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6870enum ControlRequestBodyError {
6871 UnknownOp,
6872 InvalidBody,
6873}
6874
6875#[derive(Debug, Deserialize)]
6876struct ControlOpProbe {
6877 op: String,
6878}
6879
6880const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6883
6884fn is_known_module_push_op(body: &[u8]) -> bool {
6885 serde_json::from_slice::<ControlOpProbe>(body)
6886 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6887 .unwrap_or(false)
6888}
6889
6890fn is_known_module_request_op(body: &[u8]) -> bool {
6891 serde_json::from_slice::<ControlOpProbe>(body)
6892 .map(|probe| is_module_to_subc_op(&probe.op))
6893 .unwrap_or(false)
6894}
6895
6896fn is_module_to_subc_op(op: &str) -> bool {
6897 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6898}
6899
6900fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6901 debug!(
6902 op = %op,
6903 connection_id = connection_id.get(),
6904 corr,
6905 "control dispatch"
6906 );
6907}
6908
6909fn log_slow_control_dispatch(
6910 dispatch_started_at: Option<StdInstant>,
6911 op: &'static str,
6912 connection_id: ConnectionId,
6913 corr: u64,
6914 route_open_timings: Option<&RouteOpenTimings>,
6915) {
6916 let Some(dispatch_started_at) = dispatch_started_at else {
6917 return;
6918 };
6919 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
6921 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6922 if let Some(timings) = route_open_timings {
6923 warn!(
6924 op = %op,
6925 connection_id = connection_id.get(),
6926 corr,
6927 elapsed_ms = elapsed.as_millis() as u64,
6928 module_id = ?timings.module_id,
6929 principal = %timings.principal,
6930 admission_ms = timings.admission.as_millis() as u64,
6931 project_root_ms = timings.project_root.as_millis() as u64,
6932 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
6933 bind_send_ms = timings.bind_send.as_millis() as u64,
6934 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
6935 "slow control dispatch"
6936 );
6937 return;
6938 }
6939 warn!(
6940 op = %op,
6941 connection_id = connection_id.get(),
6942 corr,
6943 elapsed_ms = elapsed.as_millis() as u64,
6944 "slow control dispatch"
6945 );
6946 }
6947}
6948
6949#[cfg(test)]
6951type TestCallerLookup =
6952 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
6953
6954fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
6956 match request {
6957 ClientControlRequest::SupervisorRestart { module_id, .. } => {
6958 Some(PrivilegedOperation::Restart { module_id })
6959 }
6960 ClientControlRequest::SupervisorSwap { module_id, .. } => {
6961 Some(PrivilegedOperation::Swap { module_id })
6962 }
6963 ClientControlRequest::SupervisorReload { module_id } => {
6964 Some(PrivilegedOperation::Reload { module_id })
6965 }
6966 ClientControlRequest::SupervisorRescan { preview } => {
6967 if *preview {
6968 None
6969 } else {
6970 Some(PrivilegedOperation::Rescan)
6971 }
6972 }
6973 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
6974 Some(PrivilegedOperation::SetEnabled {
6975 module_id,
6976 enabled: *enabled,
6977 })
6978 }
6979 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
6980 Some(PrivilegedOperation::ReleaseReserved { module_id })
6981 }
6982 ClientControlRequest::ServerDescribe {}
6983 | ClientControlRequest::CatalogList { .. }
6984 | ClientControlRequest::RouteOpen { .. }
6985 | ClientControlRequest::RoutePoll { .. }
6986 | ClientControlRequest::SupervisorList {}
6987 | ClientControlRequest::SupervisorHealth {}
6988 | ClientControlRequest::SupervisorHealthProbe { .. }
6989 | ClientControlRequest::SupervisorRoutes { .. }
6990 | ClientControlRequest::SupervisorProvenance { .. }
6991 | ClientControlRequest::SupervisorStderrTail { .. }
6992 | ClientControlRequest::SupervisorTerminals { .. }
6993 | ClientControlRequest::SupervisorSpawnSnapshot {}
6994 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
6995 }
6996}
6997
6998fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6999 match request {
7000 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7001 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7002 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7003 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7004 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7005 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7006 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7007 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7008 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7009 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7010 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7011 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7012 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7013 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7014 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7015 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7016 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7017 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7018 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7019 }
7020}
7021
7022fn principal_label(principal: &Principal) -> String {
7023 match principal {
7024 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7025 Principal::Direct => "direct".to_string(),
7026 other => format!("{other:?}"),
7027 }
7028}
7029
7030fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7031 match request {
7032 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7033 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7034 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7035 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7036 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7037 }
7038}
7039
7040fn parse_client_control_request(
7041 body: &[u8],
7042) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7043 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7044 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7045 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
7046 ControlRequestBodyError::InvalidBody
7047 }
7048 Ok(_) => ControlRequestBodyError::UnknownOp,
7049 Err(_) => ControlRequestBodyError::InvalidBody,
7050 };
7051 (err, classification)
7052 })
7053}
7054
7055fn parse_module_control_request_from_module(
7056 body: &[u8],
7057) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7058 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7059 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7060 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7061 Ok(_) => ControlRequestBodyError::UnknownOp,
7062 Err(_) => ControlRequestBodyError::InvalidBody,
7063 };
7064 (err, classification)
7065 })
7066}
7067
7068#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7069enum ProviderRoleKind {
7070 ToolProvider,
7071 PipelineStage,
7072 ManagementSurface,
7073 InternalService,
7074}
7075
7076fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7077 match role {
7078 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7079 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7080 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7081 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7082 }
7083}
7084
7085fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7086 roles.iter().map(provider_role_kind).collect()
7087}
7088
7089const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7092
7093#[derive(Debug, Default, PartialEq, Eq)]
7095struct ScopeOutcomeCounts {
7096 created: usize,
7097 replaced: usize,
7098 updated: usize,
7099 unchanged: usize,
7100 refused: usize,
7101}
7102
7103impl ScopeOutcomeCounts {
7104 fn of(results: &[ScopeRecordResult]) -> Self {
7105 let mut counts = Self::default();
7106 for result in results {
7107 let slot = match result.outcome {
7108 ScopeRecordOutcome::Created => &mut counts.created,
7109 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7110 ScopeRecordOutcome::Updated => &mut counts.updated,
7111 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7112 ScopeRecordOutcome::Refused => &mut counts.refused,
7113 };
7114 *slot += 1;
7115 }
7116 counts
7117 }
7118}
7119
7120#[cfg(test)]
7121mod scope_outcome_count_tests {
7122 use super::*;
7123
7124 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7125 ScopeRecordResult {
7126 scope_ref: "r".to_string(),
7127 scope_epoch: 1,
7128 outcome,
7129 code: None,
7130 message: None,
7131 version: None,
7132 parent_state: None,
7133 }
7134 }
7135
7136 #[test]
7139 fn every_outcome_is_counted_in_its_own_field() {
7140 let results = [
7141 result(ScopeRecordOutcome::Created),
7142 result(ScopeRecordOutcome::Created),
7143 result(ScopeRecordOutcome::Replaced),
7144 result(ScopeRecordOutcome::Updated),
7145 result(ScopeRecordOutcome::Unchanged),
7146 result(ScopeRecordOutcome::Refused),
7147 result(ScopeRecordOutcome::Refused),
7148 result(ScopeRecordOutcome::Refused),
7149 ];
7150 assert_eq!(
7151 ScopeOutcomeCounts::of(&results),
7152 ScopeOutcomeCounts {
7153 created: 2,
7154 replaced: 1,
7155 updated: 1,
7156 unchanged: 1,
7157 refused: 3,
7158 }
7159 );
7160 }
7161}
7162
7163fn capability_census_trigger(
7167 old: Option<&CapabilityDeclarations>,
7168 new: Option<&CapabilityDeclarations>,
7169) -> bool {
7170 let old_provides = old
7171 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7172 .unwrap_or_default();
7173 let old_denies = old
7174 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7175 .unwrap_or_default();
7176 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7177 provides: Vec::new(),
7178 requires: Vec::new(),
7179 must_never_reach: Vec::new(),
7180 });
7181
7182 new.provides
7183 .iter()
7184 .any(|capability| !old_provides.contains(capability))
7185 || new
7186 .must_never_reach
7187 .iter()
7188 .any(|capability| !old_denies.contains(capability))
7189}
7190
7191fn denied_capability<'a>(
7194 opening_manifest: &'a ModuleManifest,
7195 target_manifest: &ModuleManifest,
7196) -> Option<&'a str> {
7197 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7198 let target_capabilities = target_manifest.capabilities.as_ref()?;
7199 opening_capabilities
7200 .must_never_reach
7201 .iter()
7202 .find(|denied| {
7203 target_capabilities
7204 .provides
7205 .iter()
7206 .any(|provided| provided == *denied)
7207 })
7208 .map(String::as_str)
7209}
7210
7211fn catalog_update_frozen_field_message(
7212 registered: &ModuleManifest,
7213 provides: &[ProviderRole],
7214) -> Option<String> {
7215 let old_has_provides = !registered.provides.is_empty();
7216 let new_has_provides = !provides.is_empty();
7217 if old_has_provides != new_has_provides {
7218 return Some(format!(
7219 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7220 registered.module_id
7221 ));
7222 }
7223
7224 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7225 return Some(format!(
7226 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7227 registered.module_id
7228 ));
7229 }
7230
7231 let registered_concurrency = manifest_concurrency(registered);
7232 let mut candidate = registered.clone();
7233 candidate.provides = provides.to_vec();
7234 let candidate_concurrency = manifest_concurrency(&candidate);
7235 if candidate_concurrency != registered_concurrency {
7236 return Some(format!(
7237 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7238 registered.module_id, registered_concurrency, candidate_concurrency
7239 ));
7240 }
7241
7242 None
7245}
7246
7247fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7248 manifest.provides.iter().any(is_routable_role)
7249}
7250
7251fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7257 manifest
7258 .provides
7259 .iter()
7260 .find_map(|provider| match provider {
7261 ProviderRole::ToolProvider { concurrency, .. }
7262 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7263 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7264 })
7265 .unwrap_or(Concurrency::ModuleManaged)
7266}
7267
7268fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7275 let has_management_surface = manifest
7276 .provides
7277 .iter()
7278 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7279 if !has_management_surface {
7280 return false;
7281 }
7282 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7283 return false;
7284 };
7285 let Some(provides) = raw
7286 .get("manifest")
7287 .and_then(|manifest| manifest.get("provides"))
7288 .and_then(serde_json::Value::as_array)
7289 else {
7290 return false;
7291 };
7292 provides.iter().any(|role| {
7297 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7298 && role.get("concurrency").is_none()
7299 })
7300}
7301
7302fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7303 if peer_version != PROTOCOL_VERSION {
7304 return Err(format!(
7305 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7306 ));
7307 }
7308 Ok(PROTOCOL_VERSION)
7309}
7310
7311fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7312 Frame::build_with_version(
7313 response_version(frame),
7314 FrameType::Pong,
7315 frame.header.flags,
7316 0,
7317 0,
7318 frame.header.corr,
7319 Vec::new(),
7320 )
7321 .map_err(RouterError::FrameBuild)
7322}
7323
7324pub(crate) fn control_error_frame(
7325 frame: &Frame,
7326 code: &'static str,
7327 message: impl Into<String>,
7328) -> Result<Frame, RouterError> {
7329 control_error_body_frame(
7330 frame,
7331 ErrorBody {
7332 code: code.to_string(),
7333 message: message.into(),
7334 detail: None,
7335 },
7336 )
7337}
7338
7339fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7340 let body = serde_json::to_vec(&error).map_err(|err| {
7341 RouterError::backend(
7342 0,
7343 frame.header.corr,
7344 format!("failed to encode control ERROR: {err}"),
7345 )
7346 })?;
7347
7348 Frame::build_with_version(
7349 response_version(frame),
7350 FrameType::Error,
7351 control_flags(),
7352 0,
7353 0,
7354 frame.header.corr,
7355 body,
7356 )
7357 .map_err(RouterError::FrameBuild)
7358}
7359
7360fn control_response_body_frame<T: Serialize>(
7361 frame: &Frame,
7362 reply: &T,
7363 label: &'static str,
7364) -> Result<Frame, RouterError> {
7365 let body = serde_json::to_vec(reply).map_err(|err| {
7366 RouterError::backend(
7367 0,
7368 frame.header.corr,
7369 format!("failed to encode {label}: {err}"),
7370 )
7371 })?;
7372
7373 Frame::build_with_version(
7374 response_version(frame),
7375 FrameType::Response,
7376 control_flags(),
7377 0,
7378 0,
7379 frame.header.corr,
7380 body,
7381 )
7382 .map_err(RouterError::FrameBuild)
7383}
7384
7385fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7415 match err {
7416 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7417 ForwardingError::NoModuleConnection => "target_unavailable",
7418 ForwardingError::ModuleReloading { .. } => "module_reloading",
7419 ForwardingError::ClientRouteChannelExhausted { .. }
7420 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7421 ForwardingError::StaleModuleEndpoint
7422 | ForwardingError::UnknownReservation { .. }
7423 | ForwardingError::ConnectionClosing { .. }
7424 | ForwardingError::ClientEgressClosed { .. }
7425 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7426 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7429 ForwardingError::RelayCorrelationExhausted
7430 | ForwardingError::RouteOpenBuild(_)
7431 | ForwardingError::Poisoned => "forwarding_error",
7432 }
7433}
7434
7435fn response_version(frame: &Frame) -> u8 {
7436 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7437 frame.header.ver
7438 } else {
7439 PROTOCOL_VERSION
7440 }
7441}
7442
7443fn control_flags() -> Flags {
7444 Flags::new(false, Priority::Passive, false)
7445}
7446
7447fn send_goodbye_target_best_effort(
7452 counters: &DaemonCounters,
7453 target: &GoodbyeTarget,
7454 context: &'static str,
7455) {
7456 let Ok(frame) = Frame::build_with_version(
7457 target.negotiated_ver,
7458 FrameType::Goodbye,
7459 control_flags(),
7460 target.channel,
7461 target.epoch,
7462 0,
7463 Vec::new(),
7464 ) else {
7465 return;
7466 };
7467 crate::forwarding::send_module_route_goodbye(
7468 counters,
7469 &target.sink,
7470 frame,
7471 target.module_id.as_deref(),
7472 context,
7473 );
7474}
7475
7476pub(crate) fn send_route_control_pushes(
7477 forwarding: &ForwardingTable,
7478 routes: Vec<EndpointRoute>,
7479 push: ClientControlPush,
7480) {
7481 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7482 for route in routes {
7483 let target = route.goodbye_target;
7484 if let Some((existing, channels)) = targets
7485 .iter_mut()
7486 .find(|(existing, _)| existing.connection_id == target.connection_id)
7487 {
7488 debug_assert_eq!(
7489 existing.negotiated_ver, target.negotiated_ver,
7490 "one connection cannot negotiate multiple frame versions"
7491 );
7492 if !channels.contains(&target.channel) {
7493 channels.push(target.channel);
7494 }
7495 continue;
7496 }
7497 let channel = target.channel;
7498 targets.push((target, vec![channel]));
7499 }
7500 for (target, mut channels) in targets {
7501 channels.sort_unstable();
7502 let mut push = push.clone();
7503 match &mut push {
7504 ClientControlPush::RouteClosing {
7505 channels: covered, ..
7506 }
7507 | ClientControlPush::RouteClosed {
7508 channels: covered, ..
7509 } => *covered = channels,
7510 }
7511 let body = match serde_json::to_vec(&push) {
7512 Ok(body) => body,
7513 Err(err) => {
7514 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7515 continue;
7516 }
7517 };
7518 let frame = match Frame::build_with_version(
7519 target.negotiated_ver,
7520 FrameType::Push,
7521 control_flags(),
7522 0,
7523 0,
7524 0,
7525 body.clone(),
7526 ) {
7527 Ok(frame) => frame,
7528 Err(err) => {
7529 warn!(
7530 route_channel = target.channel,
7531 error = %err,
7532 "failed to build route lifecycle control PUSH frame"
7533 );
7534 continue;
7535 }
7536 };
7537 if let Err(err) = target.sink.try_send(frame) {
7538 if target.close_on_delivery_failure() {
7539 warn!(
7540 target_connection_id = target.connection_id.get(),
7541 route_channel = target.channel,
7542 error = %err,
7543 "route lifecycle control PUSH was not delivered to client; closing target connection"
7544 );
7545 let _ = forwarding.escalate_client_delivery_failure(
7546 target.connection_id,
7547 target.channel,
7548 target.epoch,
7549 CloseReason::new(
7550 "route_lifecycle_push_delivery_failed",
7551 format!(
7552 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7553 target.channel
7554 ),
7555 ),
7556 crate::forwarding::UndeliveredFrame {
7557 module_id: target.module_id.as_deref(),
7558 sink: &target.sink,
7559 },
7560 );
7561 }
7562 }
7563 }
7564}
7565
7566#[cfg(test)]
7567mod tests {
7568 mod activity_read {
7569 include!("activity_read_tests.rs");
7570 }
7571
7572 mod lifecycle_activity {
7573 include!("control_lifecycle_tests.rs");
7574 }
7575 #[cfg(unix)]
7576 #[tokio::test]
7577 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7578 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7579 let path = dir.join("subc.jsonc");
7580 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7581 "program":"/bin/sleep","args":["60"],"protocol":"none",
7582 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7583 }}}).to_string()).unwrap();
7584 let mut configured = crate::daemon_config::load(&path)
7585 .unwrap()
7586 .unwrap()
7587 .modules
7588 .pop()
7589 .unwrap();
7590 let registry = std::sync::Arc::new(crate::Registry::default());
7591 let handle = crate::SupervisorHandle::new();
7592 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7593 .with_handle(handle.clone());
7594 let module = supervisor
7595 .supervise_configured_with_health(
7596 configured.module_spec(),
7597 true,
7598 configured.health.clone(),
7599 None,
7600 configured.restart,
7601 )
7602 .unwrap();
7603 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7604 let before = module.status().unwrap().pid;
7605 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7606 configured.health.cadence = std::time::Duration::from_secs(3600);
7607 let health_only = handler
7608 .reconcile_supervised_modules(
7609 &supervisor,
7610 vec![configured.clone()],
7611 false,
7612 Principal::Direct,
7613 )
7614 .await
7615 .unwrap();
7616 assert!(
7617 health_only.changed_pending_reload.is_empty(),
7618 "health policy is already applied live"
7619 );
7620 assert_eq!(module.status().unwrap().pid, before);
7621 assert_eq!(
7622 module.configuration().unwrap().1.http,
7623 configured.health.http
7624 );
7625 configured.args = vec!["61".into()];
7626 let launch = handler
7627 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7628 .await
7629 .unwrap();
7630 assert_eq!(launch.changed_pending_reload, ["stock"]);
7631 assert_eq!(
7632 module.status().unwrap().pid,
7633 before,
7634 "a launch edit is stored until reload"
7635 );
7636 module.drain().await.unwrap();
7637 }
7638 use cortexkit_test_support::ScratchDir;
7639 use std::{
7640 collections::BTreeMap,
7641 fmt,
7642 path::PathBuf,
7643 sync::{Arc, Mutex},
7644 time::Duration,
7645 };
7646
7647 use serde_json::{json, Value};
7648 use subc_protocol::{
7649 manifest::{
7650 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7651 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7652 },
7653 session::HealthStatus,
7654 FrameType,
7655 };
7656
7657 use super::*;
7658 use crate::{
7659 forwarding::{DataRoute, DataRouteState},
7660 registry::ChannelState,
7661 router::FrameSink,
7662 stderr_tail::DEFAULT_MAX_LINE_BYTES,
7663 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7664 RouteCtx, Router,
7665 };
7666 use tokio::{
7667 sync::mpsc,
7668 time::{sleep, Instant},
7669 };
7670 use tracing::{
7671 field::{Field, Visit},
7672 Event, Subscriber,
7673 };
7674 use tracing_subscriber::{layer::Context, prelude::*, Layer};
7675
7676 fn fake_aft_stub_path() -> PathBuf {
7695 let mut path = std::env::current_exe().expect("current_exe available in tests");
7696 path.pop(); path.pop(); path.push(if cfg!(windows) {
7699 "fake-aft-stub.exe"
7700 } else {
7701 "fake-aft-stub"
7702 });
7703 assert!(
7704 path.exists(),
7705 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7706 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7707 path.display()
7708 );
7709 path
7710 }
7711
7712 fn client_retries(code: &str) -> bool {
7717 subc_protocol::error_codes::is_retryable_route_open(code)
7718 }
7719
7720 #[test]
7731 fn retryability_of_forwarding_codes_matches_the_failure() {
7732 let transient = [
7735 ForwardingError::NoModuleConnection,
7736 ForwardingError::ModuleReloading {
7737 module_id: "m".into(),
7738 },
7739 ForwardingError::StaleModuleEndpoint,
7740 ForwardingError::UnknownReservation {
7741 client_channel: 1,
7742 module_channel: 1,
7743 },
7744 ForwardingError::ConnectionClosing {
7745 connection_id: ConnectionId::new(1),
7746 },
7747 ForwardingError::ClientEgressClosed {
7748 connection_id: ConnectionId::new(1),
7749 },
7750 ForwardingError::ModuleEgressUnavailable {
7751 connection_id: ConnectionId::new(1),
7752 },
7753 ];
7754 for err in transient {
7755 let code = forwarding_error_code(&err);
7756 assert!(
7757 client_retries(code),
7758 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7759 );
7760 }
7761
7762 let permanent = [
7767 ForwardingError::ConnectionRoleConflict {
7768 connection_id: ConnectionId::new(1),
7769 },
7770 ForwardingError::ClientRouteChannelExhausted {
7771 connection_id: ConnectionId::new(1),
7772 },
7773 ForwardingError::ModuleRouteChannelExhausted {
7774 endpoint: ModuleEndpointId {
7775 connection_id: ConnectionId::new(1),
7776 generation: 1,
7777 },
7778 },
7779 ForwardingError::RelayCorrelationExhausted,
7780 ForwardingError::RouteOpenBuild("x".into()),
7781 ForwardingError::Poisoned,
7782 ];
7783 for err in permanent {
7784 let code = forwarding_error_code(&err);
7785 assert!(
7786 !client_retries(code),
7787 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7788 );
7789 }
7790 }
7791
7792 #[tokio::test]
7820 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7821 let handler = ControlHandler::default();
7822 let frame =
7823 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7824
7825 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7829 assert_eq!(
7830 stamped,
7831 Principal::Direct,
7832 "a caller that proved nothing must not be stamped as a supervised module"
7833 );
7834
7835 let forged = handler
7840 .route_open_principal(
7841 &frame,
7842 Some(ConsumerIdentity {
7843 module_id: "aft".to_string(),
7844 launch_nonce: "not-a-real-nonce".to_string(),
7845 }),
7846 )
7847 .unwrap();
7848 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7849 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7850 }
7851
7852 #[test]
7869 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7870 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7871 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7872 let ClientControlRequest::RouteOpen {
7873 consumer_identity, ..
7874 } = parsed
7875 else {
7876 panic!("route.open body must parse as RouteOpen");
7877 };
7878 assert_eq!(
7879 consumer_identity,
7880 Some(ConsumerIdentity {
7881 module_id: "aft".to_string(),
7882 launch_nonce: "n".to_string(),
7883 }),
7884 "the wire field name must reach the value route_open_principal reads"
7885 );
7886 }
7887
7888 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7889 ModuleManifest::builder(module_id, "0.1.0")
7890 .protocol_ver(protocol_ver)
7891 .provides(vec![ProviderRole::ToolProvider {
7892 tools: vec![Tool {
7893 name: "read".to_string(),
7894 description: None,
7895 execution_mode: ExecutionMode::Pure,
7896 schema: json!({"type": "object"}),
7897 }],
7898 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7899 concurrency: Concurrency::ModuleManaged,
7900 emits_push: true,
7901 sub_supervises: true,
7902 }])
7903 .build()
7904 }
7905
7906 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7907 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7908 }
7909
7910 fn hello_frame_with_control_ops(
7911 module_id: &str,
7912 protocol_ver: u8,
7913 corr: u64,
7914 control_ops: Option<Vec<String>>,
7915 ) -> Frame {
7916 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7917 }
7918
7919 fn hello_frame_with_nonce(
7920 module_id: &str,
7921 protocol_ver: u8,
7922 corr: u64,
7923 launch_nonce: Option<&str>,
7924 ) -> Frame {
7925 hello_frame_full(
7926 module_id,
7927 protocol_ver,
7928 corr,
7929 None,
7930 launch_nonce.map(ToOwned::to_owned),
7931 )
7932 }
7933
7934 fn hello_frame_full(
7935 module_id: &str,
7936 protocol_ver: u8,
7937 corr: u64,
7938 control_ops: Option<Vec<String>>,
7939 launch_nonce: Option<String>,
7940 ) -> Frame {
7941 let body = serde_json::to_vec(&ModuleHelloBody {
7942 manifest: manifest(module_id, protocol_ver),
7943 protocol_ver,
7944 control_ops,
7945 launch_nonce,
7946 })
7947 .unwrap();
7948 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7949 }
7950
7951 fn non_routable_hello_frame_with_control_ops(
7952 module_id: &str,
7953 corr: u64,
7954 control_ops: Option<Vec<String>>,
7955 ) -> Frame {
7956 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
7957 manifest.provides.clear();
7958 let body = serde_json::to_vec(&ModuleHelloBody {
7959 manifest,
7960 protocol_ver: PROTOCOL_VERSION,
7961 control_ops,
7962 launch_nonce: None,
7963 })
7964 .unwrap();
7965 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7966 }
7967
7968 fn capability_grammar_hello_frame(
7969 capabilities: Value,
7970 runtime_computed: Option<Value>,
7971 corr: u64,
7972 ) -> Frame {
7973 let mut body = serde_json::to_value(ModuleHelloBody {
7974 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
7975 protocol_ver: PROTOCOL_VERSION,
7976 control_ops: None,
7977 launch_nonce: None,
7978 })
7979 .expect("HELLO body serializes");
7980 body["manifest"]["capabilities"] = capabilities;
7981 if let Some(runtime_computed) = runtime_computed {
7982 body["runtime_computed"] = runtime_computed;
7983 }
7984 Frame::build(
7985 FrameType::Hello,
7986 control_flags(),
7987 0,
7988 0,
7989 corr,
7990 serde_json::to_vec(&body).expect("HELLO body reserializes"),
7991 )
7992 .expect("HELLO frame builds")
7993 }
7994
7995 fn channel_request(channel: u16, corr: u64) -> Frame {
7996 Frame::build(
7997 FrameType::Request,
7998 Flags::new(true, Priority::Interactive, false),
7999 channel,
8000 0,
8001 corr,
8002 b"opaque".to_vec(),
8003 )
8004 .unwrap()
8005 }
8006
8007 fn route_ctx(
8008 connection_id: ConnectionId,
8009 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8010 let (tx, rx) = mpsc::channel(8);
8011 (
8012 RouteCtx {
8013 connection_id,
8014 egress: FrameSink::new(tx),
8015 },
8016 rx,
8017 )
8018 }
8019
8020 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8021 serde_json::from_slice(&frame.body).unwrap()
8022 }
8023
8024 async fn hello_via_sink(
8029 handler: &ControlHandler,
8030 ctx: &RouteCtx,
8031 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8032 hello: Frame,
8033 ) -> Frame {
8034 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8035 assert!(
8036 replies.is_empty(),
8037 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8038 );
8039 let ack = rx
8040 .try_recv()
8041 .expect("HELLO_ACK is queued on the module sink")
8042 .frame;
8043 assert_eq!(ack.header.ty, FrameType::HelloAck);
8044 ack
8045 }
8046
8047 fn parse_error(frame: &Frame) -> Value {
8048 serde_json::from_slice(&frame.body).unwrap()
8049 }
8050
8051 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8052 serde_json::from_slice(&frame.body).unwrap()
8053 }
8054
8055 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8056 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8057 route_channel,
8058 route_epoch: 0,
8059 kind,
8060 })
8061 .unwrap();
8062 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8063 }
8064
8065 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8066 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8067 module_id: module_id.to_string(),
8068 })
8069 .unwrap();
8070 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8071 }
8072
8073 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8074 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8075 }
8076
8077 fn route_open_frame_with_consumer_capabilities(
8078 corr: u64,
8079 module_id: &str,
8080 project_root: ScratchDir,
8081 consumer_capabilities: Option<Vec<String>>,
8082 ) -> Frame {
8083 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8084 target: RouteTarget::ToolProvider {
8085 module_id: module_id.to_string(),
8086 },
8087 identity: BindIdentity::new(
8088 project_root.path().to_path_buf(),
8089 "unit".to_string(),
8090 "session".to_string(),
8091 ),
8092 consumer_identity: None,
8093 consumer_capabilities,
8094 role_versions: None,
8095 admission_facts: None,
8096 scope: None,
8097 })
8098 .unwrap();
8099 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8100 }
8101
8102 fn route_open_frame_with_role_versions(
8103 corr: u64,
8104 module_id: &str,
8105 project_root: ScratchDir,
8106 role_versions: Option<BTreeMap<String, String>>,
8107 ) -> Frame {
8108 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8109 target: RouteTarget::ToolProvider {
8110 module_id: module_id.to_string(),
8111 },
8112 identity: BindIdentity::new(
8113 project_root.path().to_path_buf(),
8114 "unit".to_string(),
8115 format!("session-{corr}"),
8116 ),
8117 consumer_identity: None,
8118 consumer_capabilities: None,
8119 role_versions,
8120 admission_facts: None,
8121 scope: None,
8122 })
8123 .unwrap();
8124 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8125 }
8126
8127 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8128 entries
8129 .iter()
8130 .map(|(role, version)| (role.to_string(), version.to_string()))
8131 .collect()
8132 }
8133
8134 fn route_open_frame_with_admission_facts(
8135 corr: u64,
8136 module_id: &str,
8137 project_root: ScratchDir,
8138 consumer_identity: Option<subc_control::ConsumerIdentity>,
8139 facts: Option<Value>,
8140 ) -> Frame {
8141 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8142 target: RouteTarget::ToolProvider {
8143 module_id: module_id.to_string(),
8144 },
8145 identity: BindIdentity::new(
8146 project_root.path().to_path_buf(),
8147 "unit".to_string(),
8148 format!("session-{corr}"),
8149 ),
8150 consumer_identity,
8151 consumer_capabilities: None,
8152 role_versions: None,
8153 admission_facts: facts,
8154 scope: None,
8155 })
8156 .unwrap();
8157 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8158 }
8159
8160 #[derive(Clone, Default)]
8161 struct EventCapture {
8162 events: Arc<Mutex<Vec<CapturedEvent>>>,
8163 }
8164
8165 #[derive(Clone, Debug)]
8166 struct CapturedEvent {
8167 target: String,
8168 level: tracing::Level,
8169 fields: BTreeMap<String, String>,
8170 }
8171
8172 impl EventCapture {
8173 fn events(&self) -> Vec<CapturedEvent> {
8174 self.events.lock().unwrap().clone()
8175 }
8176 }
8177
8178 impl<S> Layer<S> for EventCapture
8179 where
8180 S: Subscriber,
8181 {
8182 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8183 let mut visitor = EventFieldVisitor::default();
8184 event.record(&mut visitor);
8185 self.events.lock().unwrap().push(CapturedEvent {
8186 target: event.metadata().target().to_string(),
8187 level: *event.metadata().level(),
8188 fields: visitor.fields,
8189 });
8190 }
8191 }
8192
8193 #[derive(Default)]
8194 struct EventFieldVisitor {
8195 fields: BTreeMap<String, String>,
8196 }
8197
8198 impl Visit for EventFieldVisitor {
8199 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8200 self.fields
8201 .insert(field.name().to_string(), format!("{value:?}"));
8202 }
8203 }
8204
8205 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8206 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8207 status,
8208 detail: Some("warming".to_string()),
8209 metrics: Some(json!({"queue_depth": 3})),
8210 })
8211 .unwrap();
8212 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8213 }
8214
8215 fn route_bind_ack(corr: u64) -> Frame {
8216 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8217 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8218 }
8219
8220 fn unique_project_root(label: &str) -> ScratchDir {
8221 ScratchDir::new(label)
8222 }
8223
8224 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8225 match parse_route_poll(frame) {
8226 ClientControlResponse::RoutePoll {
8227 status: None,
8228 live: Some(live),
8229 ..
8230 } => assert_eq!(live, expected_live),
8231 other => panic!("unexpected route.poll response: {other:?}"),
8232 }
8233 }
8234
8235 fn bind_liveness_route(
8236 registry: &Registry,
8237 forwarding: &ForwardingTable,
8238 module_id: &str,
8239 ) -> (RouteCtx, u16, u32) {
8240 let module_connection = ConnectionId::new(101);
8241 let client_connection = ConnectionId::new(202);
8242 let registration = registry
8243 .register_with_control_ops(
8244 manifest(module_id, PROTOCOL_VERSION),
8245 PROTOCOL_VERSION,
8246 module_connection,
8247 module_baseline_control_ops(),
8248 )
8249 .unwrap();
8250 let (module_tx, _module_rx) = mpsc::channel(8);
8251 let endpoint = forwarding
8252 .register_module_connection(
8253 module_connection,
8254 module_id.to_string(),
8255 PROTOCOL_VERSION,
8256 manifest_concurrency(®istration.manifest),
8257 FrameSink::new(module_tx),
8258 )
8259 .unwrap();
8260 let (client_ctx, _client_rx) = route_ctx(client_connection);
8261 let pending = forwarding
8262 .begin_route_bind_relay_for_test(
8263 client_connection,
8264 client_ctx.egress.clone(),
8265 1,
8266 module_id,
8267 )
8268 .unwrap();
8269 assert_eq!(pending.endpoint, endpoint);
8270 let route_channel = pending.client_channel;
8271 let route_epoch = pending.client_epoch;
8272 forwarding
8273 .complete_pending_relay(
8274 module_connection,
8275 pending.corr,
8276 RouteBindRelayOutcome::Accepted,
8277 )
8278 .unwrap();
8279 (client_ctx, route_channel, route_epoch)
8280 }
8281
8282 struct FakeProcessLiveness {
8283 live: Option<bool>,
8284 }
8285
8286 impl ModuleProcessLiveness for FakeProcessLiveness {
8287 fn process_live(&self, _module_id: &str) -> Option<bool> {
8288 self.live
8289 }
8290 }
8291
8292 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8293 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8294 {
8295 let registry = Arc::new(Registry::default());
8296 let supervisor_handle = SupervisorHandle::new();
8297 let supervisor = Supervisor::new_for_test(
8298 Arc::clone(®istry),
8299 RestartPolicy::new(1, Duration::from_millis(10)),
8300 )
8301 .with_handle(supervisor_handle.clone());
8302 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8303 let module = supervisor
8304 .spawn(ModuleSpec {
8305 module_id: "stderr-tail-wire".to_string(),
8306 program: fake_aft_stub_path(),
8307 args: Vec::new(),
8308 env: vec![
8309 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8310 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8311 ],
8312 reserved: false,
8313 reserved_prefixes: Vec::new(),
8314 protocol: ModuleProtocol::Subc,
8315 overlap: Default::default(),
8316 })
8317 .unwrap();
8318
8319 let deadline = Instant::now() + Duration::from_secs(5);
8320 loop {
8321 let tail = module.stderr_tail(None, None);
8322 if tail
8323 .entries
8324 .iter()
8325 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8326 && tail.entries.iter().any(|entry| {
8327 matches!(
8328 entry,
8329 TailEntry::Line {
8330 truncated: true,
8331 ..
8332 }
8333 )
8334 })
8335 {
8336 break;
8337 }
8338 assert!(
8339 Instant::now() < deadline,
8340 "module did not produce a truncated line and restart boundary: {tail:?}"
8341 );
8342 sleep(Duration::from_millis(10)).await;
8343 }
8344
8345 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8346 let request = ClientControlRequest::SupervisorStderrTail {
8347 module_id: "stderr-tail-wire".to_string(),
8348 max_lines: None,
8349 max_bytes: None,
8350 };
8351 let frame = Frame::build(
8352 FrameType::Request,
8353 control_flags(),
8354 0,
8355 0,
8356 1,
8357 serde_json::to_vec(&request).unwrap(),
8358 )
8359 .unwrap();
8360 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8361 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8362 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8363 serde_json::from_slice(&responses[0].body).unwrap()
8364 else {
8365 panic!("expected supervisor.stderr_tail response");
8366 };
8367
8368 assert!(
8369 tail.entries
8370 .iter()
8371 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8372 "the control response lost the restart boundary"
8373 );
8374 let Some(StderrTailEntry::Line {
8375 text,
8376 truncated,
8377 at_ms,
8378 }) = tail.entries.iter().find(|entry| {
8379 matches!(
8380 entry,
8381 StderrTailEntry::Line {
8382 truncated: true,
8383 ..
8384 }
8385 )
8386 })
8387 else {
8388 panic!("the control response lost the truncated line");
8389 };
8390 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8391 assert!(*truncated);
8392 assert!(
8393 at_ms.is_some(),
8394 "the control response lost the line's capture time"
8395 );
8396 }
8397
8398 #[tokio::test(flavor = "current_thread")]
8403 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8404 let dir = ScratchDir::new("terminals-off-worker");
8405 let journal_path = dir.join("terminals.jsonl");
8406 let registry = Arc::new(Registry::default());
8407 let supervisor_handle = SupervisorHandle::new();
8408 let supervisor =
8409 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8410 .with_handle(supervisor_handle.clone())
8411 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8412 let module = supervisor
8413 .spawn(ModuleSpec {
8414 module_id: "terminal-off-worker".to_string(),
8415 program: fake_aft_stub_path(),
8416 args: Vec::new(),
8417 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8418 reserved: false,
8419 reserved_prefixes: Vec::new(),
8420 protocol: ModuleProtocol::Subc,
8421 overlap: Default::default(),
8422 })
8423 .unwrap();
8424 let deadline = Instant::now() + Duration::from_secs(5);
8425 while module.terminal_history().entries.len() != 2 {
8426 assert!(Instant::now() < deadline, "module did not record two exits");
8427 sleep(Duration::from_millis(10)).await;
8428 }
8429
8430 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8431 let handler =
8432 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8433 let frame = Frame::build(
8434 FrameType::Request,
8435 control_flags(),
8436 0,
8437 0,
8438 1,
8439 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8440 module_id: "terminal-off-worker".to_string(),
8441 })
8442 .unwrap(),
8443 )
8444 .unwrap();
8445 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8446 let spawned_at = std::time::Instant::now();
8447 let read = tokio::spawn({
8448 let handler = Arc::clone(&handler);
8449 async move { handler.handle_control_frame(&ctx, frame).await }
8450 });
8451 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8454 .await
8455 .unwrap()
8456 .expect("the history read reached its pause");
8457 let elapsed = spawned_at.elapsed();
8458 assert!(
8459 elapsed < Duration::from_secs(2) && !read.is_finished(),
8460 "this task could not run while the history read was paused \
8461 (resumed after {elapsed:?}, read finished: {})",
8462 read.is_finished()
8463 );
8464
8465 drop(release);
8466 let responses = read.await.unwrap().unwrap();
8467 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8468 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8469 panic!("expected supervisor.terminals response");
8470 };
8471 assert_eq!(terminals.entries.len(), 2);
8472 assert_eq!(terminals.journal_skipped_lines, 0);
8473 assert_eq!(terminals.journal_read_errors, 0);
8474 }
8475
8476 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8477 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8478 let registry = Arc::new(Registry::default());
8479 let supervisor_handle = SupervisorHandle::new();
8480 let supervisor =
8481 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8482 .with_handle(supervisor_handle.clone());
8483 let module = supervisor
8484 .spawn(ModuleSpec {
8485 module_id: "terminal-golden".to_string(),
8486 program: fake_aft_stub_path(),
8487 args: Vec::new(),
8488 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8489 reserved: false,
8490 reserved_prefixes: Vec::new(),
8491 protocol: ModuleProtocol::Subc,
8492 overlap: Default::default(),
8493 })
8494 .unwrap();
8495
8496 let deadline = Instant::now() + Duration::from_secs(5);
8497 while module.terminal_history().entries.len() != 2 {
8498 assert!(
8499 Instant::now() < deadline,
8500 "module did not retain two terminal exits: {:?}",
8501 module.terminal_history()
8502 );
8503 sleep(Duration::from_millis(10)).await;
8504 }
8505
8506 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8507 let request = ClientControlRequest::SupervisorTerminals {
8508 module_id: "terminal-golden".to_string(),
8509 };
8510 let frame = Frame::build(
8511 FrameType::Request,
8512 control_flags(),
8513 0,
8514 0,
8515 1,
8516 serde_json::to_vec(&request).unwrap(),
8517 )
8518 .unwrap();
8519 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8520 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8521 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8522 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8523 panic!("expected supervisor.terminals response");
8524 };
8525 assert_eq!(terminals.entries.len(), 2);
8526 assert_eq!(terminals.dropped, 0);
8527
8528 let mut rendered = serde_json::to_value(response).unwrap();
8529 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8532 for (index, entry) in rendered["entries"]
8533 .as_array_mut()
8534 .expect("terminal response entries array")
8535 .iter_mut()
8536 .enumerate()
8537 {
8538 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8539 }
8540
8541 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8542 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8543 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8544 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8545 std::fs::write(&golden_path, &serialized).unwrap();
8546 }
8547 let expected: Value =
8548 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8549 assert_eq!(rendered, expected);
8550 }
8551
8552 #[test]
8553 fn hello_registers_manifest_and_returns_ack() {
8554 let registry = Arc::new(Registry::default());
8555 let handler = ControlHandler::new(Arc::clone(®istry));
8556 let conn = ConnectionId::new(1);
8557
8558 let responses = handler
8559 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8560 .unwrap();
8561
8562 assert_eq!(responses.len(), 1);
8563 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8564 assert_eq!(responses[0].header.channel, 0);
8565 assert_eq!(responses[0].header.corr, 7);
8566 let ack = parse_ack(&responses[0]);
8567 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8568 assert!(ack
8569 .subc_capabilities
8570 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8571 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8572 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8573 assert!(ack
8574 .subc_ops
8575 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8576 assert!(ack
8577 .subc_ops
8578 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8579
8580 let registration = registry.get_module("aft").unwrap().unwrap();
8581 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8582 assert_eq!(registration.state, ChannelState::Active);
8583 assert_eq!(registration.connection_id, conn);
8584 assert_eq!(registration.control_ops, module_baseline_control_ops());
8585 }
8586
8587 #[test]
8588 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8589 let invalid_identifiers = [
8590 ("case_change", "credentials-Provider/v1"),
8591 ("leading_zero", "credentials-provider/v01"),
8592 ("trailing_hyphen", "credentials-provider-/v1"),
8593 ("consecutive_hyphens", "credentials--provider/v1"),
8594 ("uppercase", "Credentials-provider/v1"),
8595 ("missing_v", "credentials-provider/1"),
8596 ("whitespace", "credentials provider/v1"),
8597 ("zero_version", "credentials-provider/v0"),
8598 ("out_of_range_version", "credentials-provider/v4294967296"),
8599 (
8600 "overlength_name",
8601 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8602 ),
8603 ];
8604 let mut cases = invalid_identifiers
8605 .into_iter()
8606 .map(|(name, identifier)| {
8607 (
8608 format!("identifier_{name}"),
8609 "capabilities.provides[0]".to_string(),
8610 identifier.to_string(),
8611 json!({ "provides": [identifier] }),
8612 None,
8613 )
8614 })
8615 .collect::<Vec<_>>();
8616 cases.extend([
8617 (
8618 "unknown_need".to_string(),
8619 "capabilities.requires[0].need".to_string(),
8620 "deferred".to_string(),
8621 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8622 None,
8623 ),
8624 (
8625 "duplicate_provides".to_string(),
8626 "capabilities.provides[1]".to_string(),
8627 "credentials-provider/v1".to_string(),
8628 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8629 None,
8630 ),
8631 (
8632 "duplicate_must_never_reach".to_string(),
8633 "capabilities.must_never_reach[1]".to_string(),
8634 "credentials-provider/v1".to_string(),
8635 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8636 None,
8637 ),
8638 (
8639 "duplicate_requires_same_need".to_string(),
8640 "capabilities.requires[1]".to_string(),
8641 "credentials-provider/v1".to_string(),
8642 json!({ "requires": [
8643 { "capability": "credentials-provider/v1", "need": "required" },
8644 { "capability": "credentials-provider/v1", "need": "required" }
8645 ] }),
8646 None,
8647 ),
8648 (
8649 "duplicate_requires_conflicting_need".to_string(),
8650 "capabilities.requires[1]".to_string(),
8651 "credentials-provider/v1".to_string(),
8652 json!({ "requires": [
8653 { "capability": "credentials-provider/v1", "need": "required" },
8654 { "capability": "credentials-provider/v1", "need": "optional" }
8655 ] }),
8656 None,
8657 ),
8658 (
8659 "capabilities_root_pointer".to_string(),
8660 "runtime_computed[0]".to_string(),
8661 "/capabilities".to_string(),
8662 json!({}),
8663 Some(json!(["/capabilities"])),
8664 ),
8665 (
8666 "capabilities_descendant_pointer".to_string(),
8667 "runtime_computed[0]".to_string(),
8668 "/capabilities/provides".to_string(),
8669 json!({}),
8670 Some(json!(["/capabilities/provides"])),
8671 ),
8672 (
8673 "malformed_pointer_without_leading_slash".to_string(),
8674 "runtime_computed[0]".to_string(),
8675 "capabilities".to_string(),
8676 json!({}),
8677 Some(json!(["capabilities"])),
8678 ),
8679 (
8680 "malformed_pointer_escape".to_string(),
8681 "runtime_computed[0]".to_string(),
8682 "/roles/~2/tools".to_string(),
8683 json!({}),
8684 Some(json!(["/roles/~2/tools"])),
8685 ),
8686 (
8687 "unknown_capabilities_field".to_string(),
8688 "capabilities.future".to_string(),
8689 "<array>".to_string(),
8690 json!({ "future": [] }),
8691 None,
8692 ),
8693 ]);
8694
8695 for (index, (name, field, value, capabilities, runtime_computed)) in
8696 cases.into_iter().enumerate()
8697 {
8698 let registry = Arc::new(Registry::default());
8699 let handler = ControlHandler::new(Arc::clone(®istry));
8700 let response = handler
8701 .handle_control(
8702 ConnectionId::new((index + 1) as u64),
8703 capability_grammar_hello_frame(
8704 capabilities,
8705 runtime_computed,
8706 index as u64 + 1,
8707 ),
8708 )
8709 .expect("invalid HELLO returns a refusal");
8710
8711 assert_eq!(response.len(), 1, "{name} must emit one refusal");
8712 let error = parse_error(&response[0]);
8713 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8714 let message = error["message"]
8715 .as_str()
8716 .expect("error message is a string");
8717 assert!(
8718 message.contains(&field),
8719 "{name}: field missing from {message}"
8720 );
8721 assert!(
8722 message.contains(&value),
8723 "{name}: value missing from {message}"
8724 );
8725 assert_eq!(
8726 registry
8727 .active_registration_count()
8728 .expect("registry reads"),
8729 0,
8730 "{name}: refused HELLO must not create a catalog entry"
8731 );
8732 }
8733 }
8734
8735 #[test]
8736 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8737 let registry = Arc::new(Registry::default());
8738 let handler = ControlHandler::new(Arc::clone(®istry));
8739 let capabilities = json!({
8740 "provides": ["credentials-provider/v1"],
8741 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8742 "must_never_reach": ["federation-transport/v1"]
8743 });
8744 let response = handler
8745 .handle_control(
8746 ConnectionId::new(99),
8747 capability_grammar_hello_frame(
8748 capabilities.clone(),
8749 Some(json!(["/roles/0/tools"])),
8750 99,
8751 ),
8752 )
8753 .expect("valid HELLO registers");
8754 assert_eq!(response[0].header.ty, FrameType::HelloAck);
8755
8756 let request = Frame::build(
8757 FrameType::Request,
8758 control_flags(),
8759 0,
8760 0,
8761 100,
8762 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8763 .expect("catalog request serializes"),
8764 )
8765 .expect("catalog request frame builds");
8766 let response = handler
8767 .handle_catalog_list(request, None)
8768 .expect("catalog list succeeds");
8769 let ClientControlResponse::CatalogList { modules, .. } =
8770 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8771 else {
8772 panic!("catalog request must return catalog.list");
8773 };
8774 assert_eq!(modules.len(), 1);
8775 assert_eq!(
8776 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8777 capabilities
8778 );
8779 }
8780
8781 #[test]
8782 fn catalog_list_mirrors_management_operation_description() {
8783 let registry = Arc::new(Registry::default());
8784 let handler = ControlHandler::new(Arc::clone(®istry));
8785 let description = "List managed records and return their identifiers and metadata.";
8786 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8787 manifest.provides = vec![ProviderRole::ManagementSurface {
8788 operations: vec![ManagementOperation {
8789 name: "records.list".to_string(),
8790 kind: ManagementOperationKind::Query,
8791 description: Some(description.to_string()),
8792 }],
8793 config_schema: json!({"type": "object"}),
8794 observability: vec![ObservabilitySurface {
8795 name: "records.stats".to_string(),
8796 kind: ObservabilityKind::Snapshot,
8797 }],
8798 identity_scope: vec![IdentityScope::Project],
8799 concurrency: Concurrency::ModuleManaged,
8800 }];
8801 registry
8802 .register_with_control_ops(
8803 manifest,
8804 PROTOCOL_VERSION,
8805 ConnectionId::new(99),
8806 Vec::new(),
8807 )
8808 .expect("described management manifest registers");
8809
8810 let request = Frame::build(
8811 FrameType::Request,
8812 control_flags(),
8813 0,
8814 0,
8815 100,
8816 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8817 .expect("catalog request serializes"),
8818 )
8819 .expect("catalog request frame builds");
8820 let response = handler
8821 .handle_catalog_list(request, None)
8822 .expect("catalog list succeeds");
8823 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8824 assert_eq!(
8825 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8826 "catalog.list must preserve the declared operation description verbatim"
8827 );
8828 }
8829
8830 #[test]
8831 fn catalog_list_mirrors_registered_operation_delivery() {
8832 let registry = Arc::new(Registry::default());
8833 let handler = ControlHandler::new(Arc::clone(®istry));
8834 let delivery =
8835 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
8836 let mut body = serde_json::to_value(ModuleHelloBody {
8837 manifest: manifest("watch-management", PROTOCOL_VERSION),
8838 protocol_ver: PROTOCOL_VERSION,
8839 control_ops: None,
8840 launch_nonce: None,
8841 })
8842 .unwrap();
8843 body["manifest"]["provides"] = json!([{
8844 "role": "management_surface",
8845 "operations": [
8846 {"name": "status", "kind": "query"},
8847 {"name": "future", "kind": "query"},
8848 {"name": "reset", "kind": "mutate"}
8849 ],
8850 "config_schema": {}, "observability": [], "identity_scope": []
8851 }]);
8852 body["manifest"]["operation_delivery"] = delivery.clone();
8853 let hello = Frame::build(
8854 FrameType::Hello,
8855 control_flags(),
8856 0,
8857 0,
8858 99,
8859 serde_json::to_vec(&body).unwrap(),
8860 )
8861 .unwrap();
8862 let response = handler
8863 .handle_control(ConnectionId::new(99), hello)
8864 .unwrap();
8865 assert_eq!(response[0].header.ty, FrameType::HelloAck);
8866
8867 let (_, registrations) = registry.list_modules().unwrap();
8868 let registered = ®istrations[0].manifest;
8869 assert!(registered.is_watch_operation("status"));
8870 assert!(!registered.is_watch_operation("future"));
8871 assert!(!registered.is_watch_operation("reset"));
8872 assert!(!registered.is_watch_operation("missing"));
8873
8874 let request = Frame::build(
8875 FrameType::Request,
8876 control_flags(),
8877 0,
8878 0,
8879 100,
8880 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
8881 )
8882 .unwrap();
8883 let response = handler.handle_catalog_list(request, None).unwrap();
8884 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
8885 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
8886 let ClientControlResponse::CatalogList { modules, .. } =
8887 serde_json::from_slice(&response[0].body).unwrap()
8888 else {
8889 panic!("expected catalog.list");
8890 };
8891 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
8892 }
8893
8894 #[test]
8895 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8896 let registry = Arc::new(Registry::default());
8897 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8898 [("vault".to_string(), true), ("squatter".to_string(), true)],
8899 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8900 );
8901 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8902 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8903 provides: vec!["credentials-provider/v1".to_string()],
8904 requires: Vec::new(),
8905 must_never_reach: Vec::new(),
8906 });
8907 let frame = Frame::build(
8908 FrameType::Hello,
8909 control_flags(),
8910 0,
8911 0,
8912 77,
8913 serde_json::to_vec(&ModuleHelloBody {
8914 manifest: squatter,
8915 protocol_ver: PROTOCOL_VERSION,
8916 control_ops: None,
8917 launch_nonce: None,
8918 })
8919 .expect("HELLO serializes"),
8920 )
8921 .expect("HELLO frame builds");
8922 let response = handler
8923 .handle_control(ConnectionId::new(77), frame)
8924 .expect("reserved claim receives a typed refusal");
8925 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8926 assert_eq!(
8927 registry
8928 .active_registration_count()
8929 .expect("registry reads"),
8930 0,
8931 "a reserved capability refusal must not leave a catalog entry"
8932 );
8933 }
8934
8935 #[test]
8936 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8937 for settlement in ["timeout", "inconclusive", "drop"] {
8938 let breakers = RouteBindBreakers::default();
8939 let RouteBindAdmission::Admitted {
8940 guard: mut old,
8941 probe: false,
8942 } = breakers.admit("prov")
8943 else {
8944 panic!("ordinary relay admitted")
8945 };
8946 let RouteBindAdmission::Admitted {
8947 guard: mut opener, ..
8948 } = breakers.admit("prov")
8949 else {
8950 panic!("second relay admitted")
8951 };
8952 assert!(
8953 !opener
8954 .record_timeout(1, Duration::ZERO)
8955 .unwrap()
8956 .reopened_after_probe
8957 );
8958 let RouteBindAdmission::Admitted {
8959 guard: mut probe,
8960 probe: true,
8961 } = breakers.admit("prov")
8962 else {
8963 panic!("one cooldown probe admitted")
8964 };
8965 match settlement {
8966 "timeout" => assert!(
8967 !old.record_timeout(1, Duration::ZERO)
8968 .unwrap()
8969 .reopened_after_probe
8970 ),
8971 "inconclusive" => old.record_inconclusive(),
8972 "drop" => drop(old),
8973 _ => unreachable!(),
8974 }
8975 assert!(
8976 matches!(
8977 breakers.admit("prov"),
8978 RouteBindAdmission::Refused {
8979 probe_in_flight: true,
8980 ..
8981 }
8982 ),
8983 "{settlement} of a pre-open relay cannot release the real probe"
8984 );
8985 assert!(
8986 probe
8987 .record_timeout(1, Duration::ZERO)
8988 .unwrap()
8989 .reopened_after_probe
8990 );
8991 assert!(matches!(
8992 breakers.admit("prov"),
8993 RouteBindAdmission::Admitted { probe: true, .. }
8994 ));
8995 }
8996 let breakers = RouteBindBreakers::default();
8997 let admit = || match breakers.admit("prov") {
8998 RouteBindAdmission::Admitted { guard, .. } => guard,
8999 _ => panic!("relay admitted"),
9000 };
9001 admit().record_timeout(1, Duration::ZERO);
9002 let mut old_probe = admit();
9003 breakers.reset_for_new_module_connection("prov");
9004 admit().record_timeout(1, Duration::ZERO);
9005 let _new_probe = admit();
9006 old_probe.record_inconclusive();
9007 assert!(matches!(
9008 breakers.admit("prov"),
9009 RouteBindAdmission::Refused {
9010 probe_in_flight: true,
9011 ..
9012 }
9013 ));
9014 }
9015
9016 #[tokio::test]
9017 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9018 for candidate in [false, true] {
9019 let registry = Arc::new(Registry::default());
9020 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9021 [("vault".to_string(), true), ("squatter".to_string(), true)],
9022 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9023 );
9024 let conn = ConnectionId::new(77);
9025 let (ctx, mut rx) = route_ctx(conn);
9026 let initial = capability_manifest("squatter", &[], &[]);
9027 if candidate {
9028 registry
9029 .register_candidate_with_control_ops(
9030 initial.clone(),
9031 PROTOCOL_VERSION,
9032 conn,
9033 module_baseline_control_ops(),
9034 )
9035 .unwrap();
9036 handler
9037 .forwarding
9038 .register_candidate_module_connection(
9039 conn,
9040 "squatter".to_string(),
9041 PROTOCOL_VERSION,
9042 manifest_concurrency(&initial),
9043 ctx.egress.clone(),
9044 )
9045 .unwrap();
9046 } else {
9047 hello_via_sink(
9048 &handler,
9049 &ctx,
9050 &mut rx,
9051 hello_frame_with_manifest(initial.clone(), 1),
9052 )
9053 .await;
9054 }
9055 let response = handler
9056 .handle_control_frame(
9057 &ctx,
9058 catalog_update_with_capabilities_frame(
9059 2,
9060 capability_manifest("squatter", &["credentials-provider/v1"], &[])
9061 .capabilities
9062 .unwrap(),
9063 ),
9064 )
9065 .await
9066 .unwrap();
9067 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9068 assert_eq!(
9069 registry
9070 .get_module_by_connection(conn)
9071 .unwrap()
9072 .unwrap()
9073 .manifest,
9074 initial
9075 );
9076 }
9077 }
9078
9079 #[test]
9080 fn server_describe_surfaces_required_capability_verdict_fields() {
9081 let registry = Arc::new(Registry::default());
9082 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9083 [
9084 ("consumer".to_string(), true),
9085 ("provider".to_string(), false),
9086 ],
9087 BTreeMap::new(),
9088 );
9089 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9090 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9091 provides: Vec::new(),
9092 requires: vec![subc_protocol::manifest::CapabilityRequirement {
9093 capability: "credentials-provider/v1".to_string(),
9094 need: subc_protocol::manifest::CapabilityNeed::Required,
9095 }],
9096 must_never_reach: Vec::new(),
9097 });
9098 let hello = Frame::build(
9099 FrameType::Hello,
9100 control_flags(),
9101 0,
9102 0,
9103 78,
9104 serde_json::to_vec(&ModuleHelloBody {
9105 manifest: consumer,
9106 protocol_ver: PROTOCOL_VERSION,
9107 control_ops: None,
9108 launch_nonce: None,
9109 })
9110 .expect("HELLO serializes"),
9111 )
9112 .expect("HELLO frame builds");
9113 handler
9114 .handle_control(ConnectionId::new(78), hello)
9115 .expect("consumer registers");
9116 let describe = Frame::build(
9117 FrameType::Request,
9118 control_flags(),
9119 0,
9120 0,
9121 79,
9122 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
9123 .expect("request serializes"),
9124 )
9125 .expect("describe frame builds");
9126 let response = handler
9127 .handle_server_describe(describe)
9128 .expect("server.describe succeeds");
9129 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
9130 let requirement = &rendered["capability_requirements"][0];
9131 assert_eq!(requirement["consumer"], "consumer");
9132 assert_eq!(requirement["verdict"], "never_provided");
9133 assert_eq!(requirement["episode_seq"], 1);
9134 assert_eq!(requirement["config_satisfiable"], false);
9135 assert_eq!(requirement["runtime_available"], false);
9136 assert!(requirement["detail"]
9137 .as_str()
9138 .expect("detail string")
9139 .contains("credentials-provider/v1"));
9140 }
9141
9142 #[test]
9143 fn catalog_list_omits_capabilities_for_legacy_manifest() {
9144 let registry = Arc::new(Registry::default());
9145 let handler = ControlHandler::new(Arc::clone(®istry));
9146 let hello = handler
9147 .handle_control(
9148 ConnectionId::new(101),
9149 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
9150 )
9151 .expect("legacy HELLO registers");
9152 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9153
9154 let request = Frame::build(
9155 FrameType::Request,
9156 control_flags(),
9157 0,
9158 0,
9159 102,
9160 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9161 .expect("catalog request serializes"),
9162 )
9163 .expect("catalog request frame builds");
9164 let response = handler
9165 .handle_catalog_list(request, None)
9166 .expect("catalog list succeeds");
9167 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9168 assert!(
9169 body["modules"][0].get("capabilities").is_none(),
9170 "legacy manifest must retain an absent capabilities field on catalog.list"
9171 );
9172 }
9173
9174 #[test]
9175 fn hello_ack_omits_storage_when_no_storage_config() {
9176 let registry = Arc::new(Registry::default());
9177 let handler = ControlHandler::new(Arc::clone(®istry));
9178 let responses = handler
9179 .handle_control(
9180 ConnectionId::new(1),
9181 hello_frame("aft", PROTOCOL_VERSION, 7),
9182 )
9183 .unwrap();
9184 let ack = parse_ack(&responses[0]);
9185 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
9186 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
9187 }
9188
9189 #[tokio::test]
9190 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
9191 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
9192 let registry = Arc::new(Registry::default());
9193 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
9194 let responses = handler
9195 .handle_control(
9196 ConnectionId::new(1),
9197 hello_frame("aft", PROTOCOL_VERSION, 7),
9198 )
9199 .unwrap();
9200 let ack = parse_ack(&responses[0]);
9201 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
9202
9203 let described = handler
9204 .handle_control_frame(
9205 &route_ctx(ConnectionId::new(2)).0,
9206 Frame::build(
9207 FrameType::Request,
9208 control_flags(),
9209 0,
9210 0,
9211 9,
9212 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9213 )
9214 .unwrap(),
9215 )
9216 .await
9217 .unwrap();
9218 let ClientControlResponse::ServerDescribe { machine_id, .. } =
9219 serde_json::from_slice(&described[0].body).unwrap()
9220 else {
9221 panic!("server.describe answered with another shape");
9222 };
9223 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
9224 }
9225
9226 #[test]
9227 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
9228 let registry = Arc::new(Registry::default());
9231 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
9232 crate::daemon_config::StorageConfig::Sqlite {
9233 data_home: std::path::PathBuf::from("/data"),
9234 },
9235 ));
9236
9237 let responses = handler
9238 .handle_control(
9239 ConnectionId::new(1),
9240 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
9241 )
9242 .unwrap();
9243 let ack = parse_ack(&responses[0]);
9244 assert_eq!(
9245 ack.storage,
9246 Some(serde_json::json!({
9247 "module_id": "alfonso-routing",
9248 "storage_namespace": "default",
9249 "isolation": { "kind": "module" },
9250 "backend": {
9251 "backend": "sqlite",
9252 "path": "/data/cortexkit/alfonso-routing/store.db"
9253 }
9254 })),
9255 "the delivered descriptor is the module's own sqlite store path"
9256 );
9257 }
9258
9259 #[test]
9260 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
9261 let registry = Arc::new(Registry::default());
9262 let handler = ControlHandler::new(Arc::clone(®istry));
9263 let conn = ConnectionId::new(1);
9264 let responses = handler
9265 .handle_control(
9266 conn,
9267 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9268 )
9269 .unwrap();
9270 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9271 let registration = registry.get_module("aft").unwrap().unwrap();
9272 assert_eq!(registration.control_ops, module_baseline_control_ops());
9273
9274 let frame =
9275 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
9276 assert!(handler
9277 .guard_module_control_op(&frame, "aft", "route.bind")
9278 .unwrap()
9279 .is_none());
9280 let error = handler
9281 .guard_module_control_op(&frame, "aft", "test.synthetic")
9282 .unwrap()
9283 .expect("synthetic ungranted op should be rejected");
9284 assert_eq!(error.header.ty, FrameType::Error);
9285 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
9286 }
9287
9288 #[test]
9289 fn hello_control_ops_some_adds_optional_grants() {
9290 let registry = Arc::new(Registry::default());
9291 let handler = ControlHandler::new(Arc::clone(®istry));
9292 handler
9293 .handle_control(
9294 ConnectionId::new(1),
9295 hello_frame_with_control_ops(
9296 "aft",
9297 PROTOCOL_VERSION,
9298 7,
9299 Some(vec![
9300 "future.synthetic".to_string(),
9301 "route.bind".to_string(),
9302 ]),
9303 ),
9304 )
9305 .unwrap();
9306 let registration = registry.get_module("aft").unwrap().unwrap();
9307 assert_eq!(
9308 registration.control_ops,
9309 vec![
9310 "route.bind".to_string(),
9311 "route.status".to_string(),
9312 "future.synthetic".to_string(),
9313 ]
9314 );
9315 let frame =
9316 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9317 assert!(handler
9318 .guard_module_control_op(&frame, "aft", "future.synthetic")
9319 .unwrap()
9320 .is_none());
9321 }
9322
9323 #[tokio::test]
9324 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9325 let registry = Arc::new(Registry::default());
9326 let forwarding = Arc::new(ForwardingTable::default());
9327 let handler =
9328 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9329 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9330 hello_via_sink(
9331 &handler,
9332 &module_ctx,
9333 &mut module_rx,
9334 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9335 )
9336 .await;
9337
9338 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9339 let responses = handler
9340 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9341 .await
9342 .unwrap();
9343 assert_eq!(responses.len(), 1);
9344 assert_eq!(responses[0].header.ty, FrameType::Error);
9345 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9346 assert!(module_rx.try_recv().is_err());
9347 }
9348
9349 #[tokio::test]
9350 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9351 let registry = Arc::new(Registry::default());
9352 let forwarding = Arc::new(ForwardingTable::default());
9353 let handler =
9354 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9355 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9356 hello_via_sink(
9357 &handler,
9358 &module_ctx,
9359 &mut module_rx,
9360 hello_frame_with_control_ops(
9361 "aft",
9362 PROTOCOL_VERSION,
9363 7,
9364 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9365 ),
9366 )
9367 .await;
9368
9369 let project_root = unique_project_root("demux");
9370 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9371 let route_handler = handler.clone();
9372 let route_task = tokio::spawn(async move {
9373 route_handler
9374 .handle_control_frame(
9375 &route_client_ctx,
9376 route_open_frame(100, "aft", project_root),
9377 )
9378 .await
9379 .unwrap()
9380 });
9381 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9382 .await
9383 .unwrap()
9384 .unwrap();
9385 assert!(matches!(
9386 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
9387 ModuleControlRequest::RouteBind { .. }
9388 ));
9389
9390 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
9391 let health_handler = handler.clone();
9392 let health_task = tokio::spawn(async move {
9393 health_handler
9394 .handle_control_frame(
9395 &health_client_ctx,
9396 supervisor_health_probe_frame(101, "aft"),
9397 )
9398 .await
9399 .unwrap()
9400 });
9401 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9402 .await
9403 .unwrap()
9404 .unwrap();
9405 assert_eq!(
9406 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9407 ModuleControlRequest::HealthCheck {}
9408 );
9409
9410 handler
9411 .handle_control_frame(
9412 &module_ctx,
9413 health_response(health_frame.header.corr, HealthStatus::Degraded),
9414 )
9415 .await
9416 .unwrap();
9417 let health_response = health_task.await.unwrap();
9418 assert_eq!(health_response.len(), 1);
9419 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9420 ClientControlResponse::SupervisorHealthProbe {
9421 module_id,
9422 status,
9423 detail,
9424 metrics,
9425 } => {
9426 assert_eq!(module_id, "aft");
9427 assert_eq!(status, HealthStatus::Degraded);
9428 assert_eq!(detail.as_deref(), Some("warming"));
9429 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
9430 }
9431 other => panic!("unexpected health response: {other:?}"),
9432 }
9433
9434 handler
9435 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9436 .await
9437 .unwrap();
9438 let route_response = route_task.await.unwrap();
9439 assert!(route_response.is_empty());
9440 let published = route_client_rx.recv().await.unwrap();
9441 assert!(matches!(
9442 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9443 ClientControlResponse::RouteOpen { .. }
9444 ));
9445 }
9446
9447 async fn relay_route_open(
9452 handler: &ControlHandler,
9453 client_connection: ConnectionId,
9454 client_egress: &FrameSink,
9455 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9456 corr: u64,
9457 module_id: &str,
9458 project_root_label: &str,
9459 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
9460 let ctx = RouteCtx {
9461 connection_id: client_connection,
9462 egress: client_egress.clone(),
9463 };
9464 let handler = handler.clone();
9465 let project_root = unique_project_root(project_root_label);
9466 let module_id = module_id.to_string();
9467 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
9468 let task = tokio::spawn(async move {
9469 let _guard = tracing::dispatcher::set_default(&dispatch);
9470 handler
9471 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
9472 .await
9473 .unwrap()
9474 });
9475 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
9476 .await
9477 .expect("module receives the relayed route.bind")
9478 .expect("module egress is open");
9479 (task, bind.frame)
9480 }
9481
9482 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
9483 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
9484 ModuleControlRequest::RouteBind {
9485 route_channel,
9486 epoch,
9487 ..
9488 } => (route_channel, epoch),
9489 other => panic!("expected a route.bind request, got {other:?}"),
9490 }
9491 }
9492
9493 fn published_route(frame: &Frame) -> (u16, u32) {
9494 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
9495 ClientControlResponse::RouteOpen {
9496 route_channel,
9497 route_epoch,
9498 } => (route_channel, route_epoch),
9499 other => panic!("expected a route.open response, got {other:?}"),
9500 }
9501 }
9502
9503 #[tokio::test]
9523 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
9524 let registry = Arc::new(Registry::default());
9525 let forwarding = Arc::new(ForwardingTable::default());
9526 let handler =
9527 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9528
9529 let module_connection = ConnectionId::new(30);
9530 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9531 hello_via_sink(
9532 &handler,
9533 &module_ctx,
9534 &mut module_rx,
9535 hello_frame("aft", PROTOCOL_VERSION, 7),
9536 )
9537 .await;
9538
9539 let dying_client = ConnectionId::new(31);
9540 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
9541
9542 let (first_task, first_bind) = relay_route_open(
9545 &handler,
9546 dying_client,
9547 &dying_ctx.egress,
9548 &mut module_rx,
9549 100,
9550 "aft",
9551 "closing-first",
9552 )
9553 .await;
9554 handler
9555 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
9556 .await
9557 .unwrap();
9558 assert!(first_task.await.unwrap().is_empty());
9559 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
9560
9561 let (second_task, second_bind) = relay_route_open(
9563 &handler,
9564 dying_client,
9565 &dying_ctx.egress,
9566 &mut module_rx,
9567 101,
9568 "aft",
9569 "closing-second",
9570 )
9571 .await;
9572 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9573
9574 assert!(forwarding
9577 .escalate_client_delivery_failure(
9578 dying_client,
9579 first_channel,
9580 first_epoch,
9581 CloseReason::new(
9582 "module_to_client_delivery_failed",
9583 "client egress refused a module frame",
9584 ),
9585 crate::forwarding::UndeliveredFrame {
9586 module_id: None,
9587 sink: &dying_ctx.egress,
9588 },
9589 )
9590 .unwrap());
9591 assert!(!dying_ctx.egress.is_closed());
9592
9593 let ack = handler
9595 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9596 .await;
9597 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9598 if module_loop_error.is_some() {
9599 handler.cleanup_connection(module_connection).unwrap();
9603 }
9604 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9608 .await
9609 .ok()
9610 .flatten();
9611
9612 assert!(
9614 registry
9615 .get_module_by_connection(module_connection)
9616 .unwrap()
9617 .is_some(),
9618 "one client's closing connection ended the shared module connection: \
9619 {module_loop_error:?}"
9620 );
9621 let cotenant = ConnectionId::new(32);
9623 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9624 let (cotenant_task, cotenant_bind) = relay_route_open(
9625 &handler,
9626 cotenant,
9627 &cotenant_ctx.egress,
9628 &mut module_rx,
9629 102,
9630 "aft",
9631 "closing-cotenant",
9632 )
9633 .await;
9634 handler
9635 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9636 .await
9637 .unwrap();
9638 assert!(cotenant_task.await.unwrap().is_empty());
9639 let (cotenant_channel, cotenant_epoch) =
9640 published_route(&cotenant_rx.recv().await.unwrap());
9641 assert!(matches!(
9642 forwarding
9643 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9644 .unwrap(),
9645 DataRoute::Client(DataRouteState::Bound(_))
9646 ));
9647
9648 let goodbye = post_ack_module_frame
9651 .expect("module receives a GOODBYE for the abandoned route channel");
9652 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9653 assert_eq!(goodbye.header.channel, abandoned_channel);
9654 assert_eq!(goodbye.header.epoch, abandoned_epoch);
9655
9656 assert!(dying_rx.try_recv().is_err());
9660 let second_response = second_task.await.unwrap();
9661 assert_eq!(second_response.len(), 1);
9662 assert_eq!(
9663 parse_error(&second_response[0])["code"],
9664 "target_unavailable"
9665 );
9666 }
9667
9668 async fn catalog_snapshot(
9671 handler: &ControlHandler,
9672 client_ctx: &RouteCtx,
9673 corr: u64,
9674 ) -> (u64, Vec<(String, bool)>) {
9675 let body =
9676 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
9677 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
9678 let reply = handler
9679 .handle_control_frame(client_ctx, request)
9680 .await
9681 .unwrap()
9682 .pop()
9683 .unwrap();
9684 let ClientControlResponse::CatalogList {
9685 generation,
9686 modules,
9687 ..
9688 } = serde_json::from_slice(&reply.body).unwrap()
9689 else {
9690 panic!("not a catalog.list reply");
9691 };
9692 (
9693 generation,
9694 modules
9695 .into_iter()
9696 .map(|entry| (entry.module_id, entry.ready))
9697 .collect(),
9698 )
9699 }
9700
9701 fn hello_held_before_forwarding() -> (
9707 ControlHandler,
9708 Arc<std::sync::Barrier>,
9709 std::thread::JoinHandle<Vec<Frame>>,
9710 RouteCtx,
9711 mpsc::Receiver<crate::router::OutboundFrame>,
9712 ) {
9713 let hold = Arc::new(std::sync::Barrier::new(2));
9714 let handler = ControlHandler::with_forwarding(
9715 Arc::new(Registry::default()),
9716 Arc::new(ForwardingTable::default()),
9717 )
9718 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
9719 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9720 let hello = {
9721 let handler = handler.clone();
9722 let module_ctx = module_ctx.clone();
9723 std::thread::spawn(move || {
9724 handler
9725 .handle_hello(
9726 module_ctx.connection_id,
9727 Some(module_ctx.egress.clone()),
9728 hello_frame("aft", PROTOCOL_VERSION, 7),
9729 )
9730 .unwrap()
9731 })
9732 };
9733 (handler, hold, hello, module_ctx, module_rx)
9734 }
9735
9736 #[tokio::test]
9745 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
9746 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
9747 let client = ConnectionId::new(41);
9748 let (client_ctx, mut client_rx) = route_ctx(client);
9749
9750 hold.wait();
9751 assert!(handler.registry.get_module("aft").unwrap().is_some());
9753 assert!(!handler
9754 .forwarding
9755 .has_live_module_connection("aft")
9756 .unwrap());
9757 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
9758 assert!(
9759 listed.is_empty(),
9760 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
9761 );
9762 let refused = handler
9763 .handle_control_frame(
9764 &client_ctx,
9765 route_open_frame(301, "aft", unique_project_root("listed-window")),
9766 )
9767 .await
9768 .unwrap();
9769 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
9770 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
9771 let duplicate = handler
9772 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
9773 .await
9774 .unwrap();
9775 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9776
9777 hold.wait();
9778 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9779 let ack = module_rx.try_recv().unwrap().frame;
9780 assert_eq!(ack.header.ty, FrameType::HelloAck);
9781 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9782 assert_eq!(listed, vec![("aft".to_string(), true)]);
9783
9784 let (open_task, bind) = relay_route_open(
9785 &handler,
9786 client,
9787 &client_ctx.egress,
9788 &mut module_rx,
9789 303,
9790 "aft",
9791 "listed-after",
9792 )
9793 .await;
9794 assert!(handler
9795 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9796 .await
9797 .unwrap()
9798 .is_empty());
9799 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
9800 published_route(&client_rx.try_recv().unwrap().frame);
9801 }
9802
9803 #[tokio::test]
9810 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
9811 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
9812 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9813 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9814
9815 hold.wait();
9816 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
9817 assert!(listed.is_empty());
9818 assert_eq!(
9819 during, before,
9820 "the generation moved while the module was not listed"
9821 );
9822
9823 hold.wait();
9824 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9825 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9826 assert_eq!(listed, vec![("aft".to_string(), true)]);
9827 assert_eq!(
9828 after,
9829 before + 1,
9830 "the module appeared without exactly one generation bump"
9831 );
9832 }
9833
9834 #[tokio::test]
9839 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
9840 let registry = Arc::new(Registry::default());
9841 let handler = ControlHandler::with_forwarding(
9842 Arc::clone(®istry),
9843 Arc::new(ForwardingTable::default()),
9844 );
9845 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9846 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9847 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9848 drop(module_rx);
9849
9850 let replies = handler
9851 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
9852 .await
9853 .unwrap();
9854 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
9855 assert!(registry.get_module("aft").unwrap().is_none());
9856 assert!(registry
9857 .get_module_by_connection(module_ctx.connection_id)
9858 .unwrap()
9859 .is_none());
9860 assert_eq!(
9861 catalog_snapshot(&handler, &client_ctx, 301).await,
9862 (before, Vec::new())
9863 );
9864 }
9865
9866 async fn route_open_held_after_bind_send() -> (
9870 ControlHandler,
9871 Arc<tokio::sync::Barrier>,
9872 RouteCtx,
9873 mpsc::Receiver<crate::router::OutboundFrame>,
9874 ) {
9875 let hold = Arc::new(tokio::sync::Barrier::new(2));
9876 let handler = ControlHandler::with_forwarding(
9877 Arc::new(Registry::default()),
9878 Arc::new(ForwardingTable::default()),
9879 )
9880 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
9881 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9882 hello_via_sink(
9883 &handler,
9884 &module_ctx,
9885 &mut module_rx,
9886 hello_frame("aft", PROTOCOL_VERSION, 7),
9887 )
9888 .await;
9889 (handler, hold, module_ctx, module_rx)
9890 }
9891
9892 #[tokio::test]
9899 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
9900 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9901 let client = ConnectionId::new(41);
9902 let (client_ctx, mut client_rx) = route_ctx(client);
9903
9904 let (open_task, bind) = relay_route_open(
9905 &handler,
9906 client,
9907 &client_ctx.egress,
9908 &mut module_rx,
9909 200,
9910 "aft",
9911 "acked-before-mark",
9912 )
9913 .await;
9914 let (module_channel, module_epoch) = route_bind_channel(&bind);
9915 hold.wait().await;
9917
9918 assert!(handler
9920 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9921 .await
9922 .unwrap()
9923 .is_empty());
9924 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
9925
9926 hold.wait().await;
9927 assert!(
9928 open_task.await.unwrap().is_empty(),
9929 "route.open answers accepted"
9930 );
9931
9932 match module_rx.try_recv() {
9935 Err(mpsc::error::TryRecvError::Empty) => {}
9936 Ok(outbound) => panic!(
9937 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
9938 outbound.frame.header.ty,
9939 outbound.frame.header.channel,
9940 outbound.frame.header.epoch
9941 ),
9942 Err(err) => panic!("module egress closed: {err}"),
9943 }
9944
9945 let router = Router::with_control_handler(Arc::new(handler.clone()));
9948 let request = Frame::build(
9949 FrameType::Request,
9950 Flags::new(true, Priority::Interactive, false),
9951 client_channel,
9952 client_epoch,
9953 201,
9954 b"first request".to_vec(),
9955 )
9956 .unwrap();
9957 router
9958 .route_for_connection(&client_ctx, request)
9959 .await
9960 .unwrap();
9961 let delivered = module_rx.try_recv().unwrap().frame;
9962 assert_eq!(delivered.header.ty, FrameType::Request);
9963 assert_eq!(
9964 (delivered.header.channel, delivered.header.epoch),
9965 (module_channel, module_epoch)
9966 );
9967 assert_eq!(delivered.body, b"first request");
9968 }
9969
9970 #[tokio::test]
9976 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
9977 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9978 let client = ConnectionId::new(42);
9979 let (client_ctx, _client_rx) = route_ctx(client);
9980
9981 let (open_task, bind) = relay_route_open(
9982 &handler,
9983 client,
9984 &client_ctx.egress,
9985 &mut module_rx,
9986 300,
9987 "aft",
9988 "released-before-mark",
9989 )
9990 .await;
9991 let (module_channel, module_epoch) = route_bind_channel(&bind);
9992 hold.wait().await;
9993
9994 handler.cleanup_connection(client).unwrap();
9995 assert!(
9996 module_rx.try_recv().is_err(),
9997 "client teardown must not GOODBYE a bind route.open is still queueing"
9998 );
9999
10000 hold.wait().await;
10001 let response = open_task.await.unwrap();
10002 assert_eq!(response.len(), 1);
10003 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10004
10005 let goodbye = module_rx
10006 .try_recv()
10007 .expect("module is told to drop the binding the daemon abandoned")
10008 .frame;
10009 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10010 assert_eq!(
10011 (goodbye.header.channel, goodbye.header.epoch),
10012 (module_channel, module_epoch)
10013 );
10014 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10015 }
10016
10017 #[tokio::test]
10022 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
10023 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10024 let client = ConnectionId::new(43);
10025 let (client_ctx, client_rx) = route_ctx(client);
10026
10027 let (open_task, bind) = relay_route_open(
10028 &handler,
10029 client,
10030 &client_ctx.egress,
10031 &mut module_rx,
10032 400,
10033 "aft",
10034 "gone-client-before-mark",
10035 )
10036 .await;
10037 let (module_channel, module_epoch) = route_bind_channel(&bind);
10038 hold.wait().await;
10039
10040 drop(client_rx);
10041 assert!(client_ctx.egress.is_closed());
10042 handler
10043 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10044 .await
10045 .unwrap();
10046
10047 hold.wait().await;
10048 let response = open_task.await.unwrap();
10049 assert_eq!(response.len(), 1);
10050 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10051
10052 let goodbye = module_rx
10053 .try_recv()
10054 .expect("module is told to drop the binding it acked")
10055 .frame;
10056 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10057 assert_eq!(
10058 (goodbye.header.channel, goodbye.header.epoch),
10059 (module_channel, module_epoch)
10060 );
10061 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10062 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
10063 }
10064
10065 #[test]
10072 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
10073 let handler = ControlHandler::default();
10074 let module_connection = ConnectionId::new(30);
10075 let client_connection = ConnectionId::new(31);
10076
10077 handler
10078 .refuse_to_end_module_connection_for_a_client(
10079 module_connection,
10080 77,
10081 ForwardingError::ConnectionClosing {
10082 connection_id: client_connection,
10083 },
10084 )
10085 .expect("a closing client must never end the module connection");
10086
10087 assert!(matches!(
10088 handler.refuse_to_end_module_connection_for_a_client(
10089 module_connection,
10090 78,
10091 ForwardingError::ConnectionClosing {
10092 connection_id: module_connection,
10093 },
10094 ),
10095 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
10096 connection_id
10097 })) if connection_id == module_connection
10098 ));
10099 assert!(matches!(
10100 handler.refuse_to_end_module_connection_for_a_client(
10101 module_connection,
10102 79,
10103 ForwardingError::Poisoned,
10104 ),
10105 Err(RouterError::Forwarding(ForwardingError::Poisoned))
10106 ));
10107 assert!(matches!(
10108 handler.refuse_to_end_module_connection_for_a_client(
10109 module_connection,
10110 80,
10111 ForwardingError::StaleModuleEndpoint,
10112 ),
10113 Err(RouterError::Forwarding(
10114 ForwardingError::StaleModuleEndpoint
10115 ))
10116 ));
10117 }
10118
10119 #[tokio::test]
10132 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
10133 let registry = Arc::new(Registry::default());
10134 let forwarding = Arc::new(ForwardingTable::default());
10135 let supervisor = SupervisorHandle::new();
10136 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10137 let handler =
10138 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10139 .with_supervisor(supervisor);
10140
10141 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10142 hello_via_sink(
10143 &handler,
10144 &target_ctx,
10145 &mut target_rx,
10146 hello_frame("target", PROTOCOL_VERSION, 1),
10147 )
10148 .await;
10149
10150 let wrong_nonce = handler
10154 .handle_control_frame(
10155 &route_ctx(ConnectionId::new(91)).0,
10156 route_open_frame_with_admission_facts(
10157 20,
10158 "target",
10159 unique_project_root("admission-facts"),
10160 Some(subc_control::ConsumerIdentity {
10161 module_id: "fed".to_string(),
10162 launch_nonce: "not-the-real-nonce".to_string(),
10163 }),
10164 None,
10165 ),
10166 )
10167 .await
10168 .unwrap();
10169 assert_eq!(
10170 parse_error(&wrong_nonce[0])["code"],
10171 "bad_consumer_identity",
10172 "a mismatched launch nonce must be refused, not stamped Reserved"
10173 );
10174
10175 let never_spawned = handler
10179 .handle_control_frame(
10180 &route_ctx(ConnectionId::new(92)).0,
10181 route_open_frame_with_admission_facts(
10182 21,
10183 "target",
10184 unique_project_root("admission-facts"),
10185 Some(subc_control::ConsumerIdentity {
10186 module_id: "never-spawned".to_string(),
10187 launch_nonce: "any-nonce".to_string(),
10188 }),
10189 None,
10190 ),
10191 )
10192 .await
10193 .unwrap();
10194 assert_eq!(
10195 parse_error(&never_spawned[0])["code"],
10196 "bad_consumer_identity",
10197 "an unspawned module_id must be refused rather than accepted for lack of a record"
10198 );
10199 }
10200
10201 #[tokio::test]
10224 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
10225 let registry = Arc::new(Registry::default());
10226 let forwarding = Arc::new(ForwardingTable::default());
10227 let supervisor = SupervisorHandle::new();
10228 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10229 let handler =
10230 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10231 .with_supervisor(supervisor);
10232
10233 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
10234 hello_via_sink(
10235 &handler,
10236 &target_ctx,
10237 &mut target_rx,
10238 hello_frame("target", PROTOCOL_VERSION, 1),
10239 )
10240 .await;
10241
10242 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
10243 let route_handler = handler.clone();
10244 let route_task = tokio::spawn(async move {
10245 route_handler
10246 .handle_control_frame(
10247 &client_ctx,
10248 route_open_frame_with_admission_facts(
10249 30,
10250 "target",
10251 unique_project_root("admission-facts"),
10252 Some(subc_control::ConsumerIdentity {
10253 module_id: "fed".to_string(),
10254 launch_nonce: "fed-nonce".to_string(),
10255 }),
10256 None,
10257 ),
10258 )
10259 .await
10260 .unwrap()
10261 });
10262
10263 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10270 .await
10271 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
10272 .expect("module control channel closed before route.bind");
10273 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10274 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10275 panic!("expected route.bind")
10276 };
10277 assert_eq!(
10278 principal,
10279 Some(Principal::Reserved {
10280 module_id: "fed".to_string()
10281 }),
10282 "a correctly attested consumer must be stamped Reserved for its own id"
10283 );
10284
10285 handler
10286 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10287 .await
10288 .unwrap();
10289 assert!(route_task.await.unwrap().is_empty());
10290 assert!(
10291 matches!(
10292 serde_json::from_slice::<ClientControlResponse>(
10293 &client_rx.recv().await.unwrap().body
10294 )
10295 .unwrap(),
10296 ClientControlResponse::RouteOpen { .. }
10297 ),
10298 "the route must actually open, not merely avoid an error"
10299 );
10300 }
10301
10302 #[tokio::test(start_paused = true)]
10303 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10304 let registry = Arc::new(Registry::default());
10305 let forwarding = Arc::new(ForwardingTable::default());
10306 let supervisor = SupervisorHandle::new();
10307 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10308 let handler =
10309 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10310 .with_supervisor(supervisor);
10311
10312 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10313 hello_via_sink(
10314 &handler,
10315 &target_ctx,
10316 &mut target_rx,
10317 hello_frame("target", PROTOCOL_VERSION, 1),
10318 )
10319 .await;
10320
10321 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10322 let direct_handler = handler.clone();
10323 let direct_open = tokio::spawn(async move {
10324 direct_handler
10325 .handle_control_frame(
10326 &direct_ctx,
10327 route_open_frame(2, "target", unique_project_root("route-census-direct")),
10328 )
10329 .await
10330 .unwrap()
10331 });
10332 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10333 .await
10334 .expect("no direct route.bind within 5s")
10335 .expect("target control channel closed before direct route.bind");
10336 handler
10337 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10338 .await
10339 .unwrap();
10340 assert!(direct_open.await.unwrap().is_empty());
10341 let _ = direct_rx.recv().await.unwrap();
10342
10343 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10344 let reserved_handler = handler.clone();
10345 let reserved_open = tokio::spawn(async move {
10346 reserved_handler
10347 .handle_control_frame(
10348 &reserved_ctx,
10349 route_open_frame_with_admission_facts(
10350 3,
10351 "target",
10352 unique_project_root("admission-facts"),
10353 Some(ConsumerIdentity {
10354 module_id: "fed".to_string(),
10355 launch_nonce: "fed-nonce".to_string(),
10356 }),
10357 None,
10358 ),
10359 )
10360 .await
10361 .unwrap()
10362 });
10363 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10364 .await
10365 .expect("no reserved route.bind within 5s")
10366 .expect("target control channel closed before reserved route.bind");
10367 handler
10368 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10369 .await
10370 .unwrap();
10371 assert!(reserved_open.await.unwrap().is_empty());
10372 let _ = reserved_rx.recv().await.unwrap();
10373
10374 forwarding
10375 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10376 .unwrap();
10377 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
10378 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
10379 module_id: Some("target".to_string()),
10380 })
10381 .unwrap();
10382 let census_frame =
10383 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
10384 let response = handler
10385 .handle_control_frame(&census_ctx, census_frame)
10386 .await
10387 .unwrap()
10388 .pop()
10389 .unwrap();
10390 let actual: Value = serde_json::from_slice(&response.body).unwrap();
10391 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
10392 assert!(matches!(
10393 decoded,
10394 ClientControlResponse::SupervisorRoutes { .. }
10395 ));
10396 let routes = actual["modules"][0]["routes"].as_array().unwrap();
10397 assert_eq!(routes.len(), 2);
10398 assert!(routes.iter().all(|route| route["draining"] == true));
10399 assert!(
10402 routes.iter().all(|route| route["drain_reason"] == "reload"),
10403 "draining routes must name the drain's reason: {routes:?}"
10404 );
10405 assert!(routes.iter().any(|route| {
10406 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
10407 }));
10408 assert!(routes.iter().any(|route| {
10409 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
10410 }));
10411
10412 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10413 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
10414 if std::env::var_os("UPDATE_GOLDEN").is_some() {
10415 std::fs::write(
10416 &golden_path,
10417 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
10418 )
10419 .unwrap();
10420 }
10421 let expected: Value =
10422 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
10423 assert_eq!(actual, expected);
10424 }
10425
10426 async fn query_live_roots(
10427 handler: &ControlHandler,
10428 module_ctx: &RouteCtx,
10429 ) -> ModuleControlResponseToModule {
10430 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
10431 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
10432 let response = handler
10433 .handle_control_frame(module_ctx, frame)
10434 .await
10435 .unwrap()
10436 .pop()
10437 .unwrap();
10438 serde_json::from_slice(&response.body).unwrap()
10439 }
10440
10441 #[tokio::test(start_paused = true)]
10442 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
10443 let registry = Arc::new(Registry::default());
10444 let forwarding = Arc::new(ForwardingTable::default());
10445 let handler = ControlHandler::with_forwarding(registry, forwarding);
10446 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
10447 hello_via_sink(
10448 &handler,
10449 &target_ctx,
10450 &mut target_rx,
10451 hello_frame("target", PROTOCOL_VERSION, 1),
10452 )
10453 .await;
10454 let root = unique_project_root("live-roots-known");
10455 let path = ProjectRootId::from_path_allowing_missing(root.path())
10456 .unwrap()
10457 .as_path()
10458 .to_path_buf();
10459 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
10460 let open_handler = handler.clone();
10461 let opened = tokio::spawn(async move {
10462 open_handler
10463 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
10464 .await
10465 .unwrap()
10466 });
10467 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10468 .await
10469 .unwrap()
10470 .unwrap();
10471 handler
10472 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
10473 .await
10474 .unwrap();
10475 assert!(opened.await.unwrap().is_empty());
10476 let _ = client_rx.recv().await.unwrap();
10477
10478 let root = unique_project_root("live-roots-pending");
10479 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
10480 .unwrap()
10481 .as_path()
10482 .to_path_buf();
10483 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
10484 let open_handler = handler.clone();
10485 let pending = tokio::spawn(async move {
10486 open_handler
10487 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
10488 .await
10489 .unwrap()
10490 });
10491 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10492 .await
10493 .unwrap()
10494 .unwrap();
10495 let actual = query_live_roots(&handler, &target_ctx).await;
10496 let ModuleControlResponseToModule::LiveRoots {
10497 roots,
10498 unknown_root_bindings,
10499 total_bindings,
10500 } = actual
10501 else {
10502 panic!("expected live roots")
10503 };
10504 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
10505 assert_eq!(unknown_root_bindings, 0);
10506 assert_eq!(
10507 roots.len(),
10508 2,
10509 "root-known arm must retain each canonical root"
10510 );
10511 assert_eq!(
10512 total_bindings,
10513 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10514 );
10515 let counts = roots
10516 .iter()
10517 .map(|root| (root.project_root.clone(), root.bound, root.pending))
10518 .collect::<Vec<_>>();
10519 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
10520 expected.sort_by(|a, b| a.0.cmp(&b.0));
10521 assert_eq!(
10522 counts, expected,
10523 "roots must sort by path and count pending separately"
10524 );
10525 handler
10526 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
10527 .await
10528 .unwrap();
10529 assert!(pending.await.unwrap().is_empty());
10530 }
10531
10532 #[tokio::test(start_paused = true)]
10533 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
10534 let forwarding = Arc::new(ForwardingTable::default());
10535 let handler =
10536 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10537 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
10538 hello_via_sink(
10539 &handler,
10540 &target_ctx,
10541 &mut target_rx,
10542 hello_frame("target", PROTOCOL_VERSION, 1),
10543 )
10544 .await;
10545 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
10546 let pending = forwarding
10547 .begin_route_bind_relay_for_test(
10548 client_ctx.connection_id,
10549 client_ctx.egress.clone(),
10550 2,
10551 "target",
10552 )
10553 .unwrap();
10554 forwarding
10555 .complete_pending_relay(
10556 target_ctx.connection_id,
10557 pending.corr,
10558 RouteBindRelayOutcome::Accepted,
10559 )
10560 .unwrap();
10561 let actual = query_live_roots(&handler, &target_ctx).await;
10562 let ModuleControlResponseToModule::LiveRoots {
10563 roots,
10564 unknown_root_bindings,
10565 total_bindings,
10566 } = actual
10567 else {
10568 panic!("expected live roots")
10569 };
10570 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
10571 assert_eq!(
10572 unknown_root_bindings, 1,
10573 "unknown-root arm must not read as no bindings"
10574 );
10575 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
10576 assert_eq!(
10577 total_bindings,
10578 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10579 );
10580 }
10581
10582 #[tokio::test(start_paused = true)]
10590 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
10591 let forwarding = Arc::new(ForwardingTable::default());
10592 let handler =
10593 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10594 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
10595 let replies = handler
10596 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
10597 .await
10598 .unwrap();
10599 let queued_by_hello = module_rx.len();
10600
10601 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
10602 let open_handler = handler.clone();
10603 let open = tokio::spawn(async move {
10604 open_handler
10605 .handle_control_frame(
10606 &client_ctx,
10607 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
10608 )
10609 .await
10610 .unwrap()
10611 });
10612 let mut spins = 0;
10614 while module_rx.len() == queued_by_hello {
10615 spins += 1;
10616 assert!(spins < 10_000, "route.open never queued a route.bind");
10617 tokio::task::yield_now().await;
10618 }
10619
10620 for reply in replies {
10622 module_ctx.egress.send(reply).await.unwrap();
10623 }
10624
10625 let first = module_rx.recv().await.unwrap().frame;
10626 assert_eq!(
10627 first.header.ty,
10628 FrameType::HelloAck,
10629 "the first frame a registering module reads must be its HELLO_ACK"
10630 );
10631 assert_eq!(first.header.corr, 7);
10632 let second = module_rx.recv().await.unwrap().frame;
10633 assert_eq!(second.header.ty, FrameType::Request);
10634 assert!(
10635 matches!(
10636 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
10637 ModuleControlRequest::RouteBind { .. }
10638 ),
10639 "the route.bind follows the ack"
10640 );
10641 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
10642
10643 handler
10644 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
10645 .await
10646 .unwrap();
10647 assert!(open.await.unwrap().is_empty());
10648 let _ = client_rx.recv().await.unwrap();
10649 }
10650
10651 #[tokio::test(start_paused = true)]
10652 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
10653 let handler = ControlHandler::with_forwarding(
10654 Arc::new(Registry::default()),
10655 Arc::new(ForwardingTable::default()),
10656 );
10657 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
10658 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
10659 hello_via_sink(
10660 &handler,
10661 &first_ctx,
10662 &mut first_rx,
10663 hello_frame("first", PROTOCOL_VERSION, 1),
10664 )
10665 .await;
10666 hello_via_sink(
10667 &handler,
10668 &second_ctx,
10669 &mut second_rx,
10670 hello_frame("second", PROTOCOL_VERSION, 2),
10671 )
10672 .await;
10673 let root = unique_project_root("second-only");
10674 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
10675 let cloned = handler.clone();
10676 let open = tokio::spawn(async move {
10677 cloned
10678 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
10679 .await
10680 .unwrap()
10681 });
10682 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
10683 .await
10684 .unwrap()
10685 .unwrap();
10686 let first = query_live_roots(&handler, &first_ctx).await;
10687 let second = query_live_roots(&handler, &second_ctx).await;
10688 assert!(
10689 matches!(
10690 first,
10691 ModuleControlResponseToModule::LiveRoots {
10692 total_bindings: 0,
10693 ..
10694 }
10695 ),
10696 "cross-module scope must not expose another module's roots"
10697 );
10698 assert!(
10699 matches!(
10700 second,
10701 ModuleControlResponseToModule::LiveRoots {
10702 total_bindings: 1,
10703 ..
10704 }
10705 ),
10706 "second module must see its pending route"
10707 );
10708 handler
10709 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
10710 .await
10711 .unwrap();
10712 assert!(open.await.unwrap().is_empty());
10713 }
10714
10715 #[tokio::test(start_paused = true)]
10716 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
10717 let handler = ControlHandler::with_forwarding(
10718 Arc::new(Registry::default()),
10719 Arc::new(ForwardingTable::default()),
10720 );
10721 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
10722 hello_via_sink(
10723 &handler,
10724 &target_ctx,
10725 &mut target_rx,
10726 hello_frame("target", PROTOCOL_VERSION, 1),
10727 )
10728 .await;
10729 let actual = query_live_roots(&handler, &target_ctx).await;
10730 let ModuleControlResponseToModule::LiveRoots {
10731 roots,
10732 unknown_root_bindings,
10733 total_bindings,
10734 } = actual
10735 else {
10736 panic!("expected live roots")
10737 };
10738 assert!(roots.is_empty());
10739 assert_eq!(unknown_root_bindings, 0);
10740 assert_eq!(total_bindings, 0);
10741 assert_eq!(
10742 total_bindings,
10743 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10744 );
10745 }
10746
10747 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
10754 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10755 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
10756 let text = std::fs::read_to_string(&path)
10757 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
10758 let vectors: Vec<(String, Value)> = text
10759 .lines()
10760 .filter(|line| !line.trim().is_empty())
10761 .map(|line| {
10762 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
10763 let id = entry["corpus_id"]
10764 .as_str()
10765 .expect("every vector carries a corpus_id")
10766 .to_string();
10767 (id, entry["package"].clone())
10768 })
10769 .collect();
10770 assert_eq!(
10774 vectors.len(),
10775 3,
10776 "vendored fed corpus changed size; re-sync from subc-federation"
10777 );
10778
10779 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
10794 let richest = vectors
10795 .iter()
10796 .filter_map(|(_, package)| package.as_object())
10797 .map(|object| {
10798 object
10799 .keys()
10800 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
10801 .count()
10802 })
10803 .max()
10804 .unwrap_or(0);
10805 assert!(
10806 richest >= 2,
10807 "vendored corpus no longer carries a package with unmodelled fields, \
10808 so the relay test can no longer distinguish a verbatim relay from a lossy one"
10809 );
10810
10811 vectors
10812 }
10813
10814 #[tokio::test]
10822 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
10823 for (corpus_id, package) in fed_admission_facts_vectors() {
10824 let registry = Arc::new(Registry::default());
10825 let forwarding = Arc::new(ForwardingTable::default());
10826 let supervisor = SupervisorHandle::new();
10827 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10828 let handler =
10829 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10830 .with_supervisor(supervisor)
10831 .with_admission_facts_config(
10832 Some("fed".to_string()),
10833 Some(vec!["target".to_string()]),
10834 );
10835
10836 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10837 hello_via_sink(
10838 &handler,
10839 &target_ctx,
10840 &mut target_rx,
10841 hello_frame("target", PROTOCOL_VERSION, 1),
10842 )
10843 .await;
10844
10845 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
10846 let route_handler = handler.clone();
10847 let expected = package.clone();
10848 let route_task = tokio::spawn(async move {
10849 route_handler
10850 .handle_control_frame(
10851 &client_ctx,
10852 route_open_frame_with_admission_facts(
10853 20,
10854 "target",
10855 unique_project_root("admission-facts"),
10856 Some(subc_control::ConsumerIdentity {
10857 module_id: "fed".to_string(),
10858 launch_nonce: "fed-nonce".to_string(),
10859 }),
10860 Some(package),
10861 ),
10862 )
10863 .await
10864 .unwrap()
10865 });
10866
10867 let bind_frame = target_rx.recv().await.unwrap();
10868 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10869 let ModuleControlRequest::RouteBind {
10870 admission_facts, ..
10871 } = bind
10872 else {
10873 panic!("{corpus_id}: expected route.bind")
10874 };
10875 assert_eq!(
10876 admission_facts,
10877 Some(expected),
10878 "{corpus_id}: relay must not add, drop or reshape any field"
10879 );
10880
10881 handler
10882 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10883 .await
10884 .unwrap();
10885 route_task.await.unwrap();
10886 }
10887 }
10888
10889 #[tokio::test]
10890 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
10891 let registry = Arc::new(Registry::default());
10892 let forwarding = Arc::new(ForwardingTable::default());
10893 let supervisor = SupervisorHandle::new();
10894 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10895 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
10896 let handler =
10897 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10898 .with_supervisor(supervisor)
10899 .with_admission_facts_config(
10900 Some("fed".to_string()),
10901 Some(vec!["target".to_string()]),
10902 );
10903
10904 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
10905 hello_via_sink(
10906 &handler,
10907 &target_ctx,
10908 &mut target_rx,
10909 hello_frame("target", PROTOCOL_VERSION, 1),
10910 )
10911 .await;
10912 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
10913 hello_via_sink(
10914 &handler,
10915 &other_ctx,
10916 &mut other_rx,
10917 hello_frame("other", PROTOCOL_VERSION, 2),
10918 )
10919 .await;
10920
10921 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
10922 let expected_facts = facts.clone();
10923 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
10924 let route_handler = handler.clone();
10925 let route_task = tokio::spawn(async move {
10926 route_handler
10927 .handle_control_frame(
10928 &client_ctx,
10929 route_open_frame_with_admission_facts(
10930 10,
10931 "target",
10932 unique_project_root("admission-facts"),
10933 Some(subc_control::ConsumerIdentity {
10934 module_id: "fed".to_string(),
10935 launch_nonce: "fed-nonce".to_string(),
10936 }),
10937 Some(facts.clone()),
10938 ),
10939 )
10940 .await
10941 .unwrap()
10942 });
10943 let bind_frame = target_rx.recv().await.unwrap();
10944 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10945 let ModuleControlRequest::RouteBind {
10946 admission_facts, ..
10947 } = bind
10948 else {
10949 panic!("expected route.bind")
10950 };
10951 assert_eq!(admission_facts, Some(expected_facts));
10952 handler
10953 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10954 .await
10955 .unwrap();
10956 assert!(route_task.await.unwrap().is_empty());
10957 assert!(matches!(
10958 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
10959 .unwrap(),
10960 ClientControlResponse::RouteOpen { .. }
10961 ));
10962
10963 let direct = handler
10964 .handle_control_frame(
10965 &route_ctx(ConnectionId::new(73)).0,
10966 route_open_frame_with_admission_facts(
10967 11,
10968 "target",
10969 unique_project_root("admission-facts"),
10970 None,
10971 Some(json!({"x": 1})),
10972 ),
10973 )
10974 .await
10975 .unwrap();
10976 assert_eq!(
10977 parse_error(&direct[0])["code"],
10978 "admission_facts_not_permitted"
10979 );
10980
10981 let different_reserved = handler
10982 .handle_control_frame(
10983 &route_ctx(ConnectionId::new(77)).0,
10984 route_open_frame_with_admission_facts(
10985 15,
10986 "target",
10987 unique_project_root("admission-facts"),
10988 Some(subc_control::ConsumerIdentity {
10989 module_id: "other".to_string(),
10990 launch_nonce: "other-nonce".to_string(),
10991 }),
10992 Some(json!({"x": 1})),
10993 ),
10994 )
10995 .await
10996 .unwrap();
10997 assert_eq!(
10998 parse_error(&different_reserved[0])["code"],
10999 "admission_facts_not_permitted"
11000 );
11001
11002 let other_target = handler
11003 .handle_control_frame(
11004 &route_ctx(ConnectionId::new(74)).0,
11005 route_open_frame_with_admission_facts(
11006 12,
11007 "other",
11008 unique_project_root("admission-facts"),
11009 Some(subc_control::ConsumerIdentity {
11010 module_id: "fed".to_string(),
11011 launch_nonce: "fed-nonce".to_string(),
11012 }),
11013 Some(json!({"x": 1})),
11014 ),
11015 )
11016 .await
11017 .unwrap();
11018 assert_eq!(
11019 parse_error(&other_target[0])["code"],
11020 "admission_facts_target_not_allowed"
11021 );
11022
11023 let nonexistent = handler
11024 .handle_control_frame(
11025 &route_ctx(ConnectionId::new(75)).0,
11026 route_open_frame_with_admission_facts(
11027 13,
11028 "missing",
11029 unique_project_root("admission-facts"),
11030 None,
11031 Some(json!({"x": 1})),
11032 ),
11033 )
11034 .await
11035 .unwrap();
11036 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
11037
11038 let described = handler
11039 .handle_control_frame(
11040 &route_ctx(ConnectionId::new(76)).0,
11041 Frame::build(
11042 FrameType::Request,
11043 control_flags(),
11044 0,
11045 0,
11046 14,
11047 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11048 )
11049 .unwrap(),
11050 )
11051 .await
11052 .unwrap();
11053 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11054 serde_json::from_slice(&described[0].body).unwrap()
11055 else {
11056 panic!("expected server.describe response")
11057 };
11058 assert!(capabilities
11059 .iter()
11060 .any(|cap| cap == "admission_facts_relay_v1"));
11061 }
11062
11063 #[tokio::test]
11064 async fn admission_facts_without_configured_carrier_are_rejected() {
11065 let registry = Arc::new(Registry::default());
11066 let forwarding = Arc::new(ForwardingTable::default());
11067 let handler = ControlHandler::with_forwarding(registry, forwarding);
11068 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
11069 hello_via_sink(
11070 &handler,
11071 &target_ctx,
11072 &mut target_rx,
11073 hello_frame("target", PROTOCOL_VERSION, 1),
11074 )
11075 .await;
11076
11077 let responses = handler
11078 .handle_control_frame(
11079 &route_ctx(ConnectionId::new(79)).0,
11080 route_open_frame_with_admission_facts(
11081 16,
11082 "target",
11083 unique_project_root("admission-facts"),
11084 None,
11085 Some(json!({"x": 1})),
11086 ),
11087 )
11088 .await
11089 .unwrap();
11090 assert_eq!(
11091 parse_error(&responses[0])["code"],
11092 "admission_facts_not_permitted"
11093 );
11094 }
11095
11096 #[tokio::test]
11097 async fn route_open_relays_consumer_capabilities_verbatim() {
11098 let registry = Arc::new(Registry::default());
11099 let forwarding = Arc::new(ForwardingTable::default());
11100 let handler =
11101 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11102 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
11103 hello_via_sink(
11104 &handler,
11105 &module_ctx,
11106 &mut module_rx,
11107 hello_frame("aft", PROTOCOL_VERSION, 7),
11108 )
11109 .await;
11110
11111 let expected = vec!["elicitation".to_string(), "roots".to_string()];
11112 let expected_for_request = expected.clone();
11113 let project_root = unique_project_root("consumer-capabilities-present");
11114 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
11115 let route_handler = handler.clone();
11116 let route_task = tokio::spawn(async move {
11117 route_handler
11118 .handle_control_frame(
11119 &client_ctx,
11120 route_open_frame_with_consumer_capabilities(
11121 401,
11122 "aft",
11123 project_root,
11124 Some(expected_for_request),
11125 ),
11126 )
11127 .await
11128 .unwrap()
11129 });
11130 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11131 .await
11132 .unwrap()
11133 .unwrap();
11134 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11135 let ModuleControlRequest::RouteBind {
11136 consumer_capabilities,
11137 ..
11138 } = bind
11139 else {
11140 panic!("expected route.bind request, got {bind:?}");
11141 };
11142 assert_eq!(consumer_capabilities, Some(expected.clone()));
11143
11144 handler
11145 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11146 .await
11147 .unwrap();
11148 let route_response = route_task.await.unwrap();
11149 assert!(route_response.is_empty());
11150 let published = client_rx.recv().await.unwrap();
11151 assert!(matches!(
11152 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11153 ClientControlResponse::RouteOpen { .. }
11154 ));
11155 }
11156
11157 #[tokio::test]
11158 async fn route_open_without_consumer_capabilities_relays_none() {
11159 let registry = Arc::new(Registry::default());
11160 let forwarding = Arc::new(ForwardingTable::default());
11161 let handler =
11162 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11163 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
11164 hello_via_sink(
11165 &handler,
11166 &module_ctx,
11167 &mut module_rx,
11168 hello_frame("aft", PROTOCOL_VERSION, 7),
11169 )
11170 .await;
11171
11172 let project_root = unique_project_root("consumer-capabilities-absent");
11173 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
11174 let route_handler = handler.clone();
11175 let route_task = tokio::spawn(async move {
11176 route_handler
11177 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
11178 .await
11179 .unwrap()
11180 });
11181 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11182 .await
11183 .unwrap()
11184 .unwrap();
11185 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11186 let ModuleControlRequest::RouteBind {
11187 consumer_capabilities,
11188 ..
11189 } = bind
11190 else {
11191 panic!("expected route.bind request, got {bind:?}");
11192 };
11193 assert_eq!(consumer_capabilities, None);
11194
11195 handler
11196 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11197 .await
11198 .unwrap();
11199 let route_response = route_task.await.unwrap();
11200 assert!(route_response.is_empty());
11201 let published = client_rx.recv().await.unwrap();
11202 assert!(matches!(
11203 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11204 ClientControlResponse::RouteOpen { .. }
11205 ));
11206 }
11207
11208 async fn bind_role_versions_for(
11212 sent: Option<BTreeMap<String, String>>,
11213 connection: u64,
11214 ) -> Option<BTreeMap<String, String>> {
11215 let registry = Arc::new(Registry::default());
11216 let forwarding = Arc::new(ForwardingTable::default());
11217 let handler =
11218 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11219 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
11220 hello_via_sink(
11221 &handler,
11222 &module_ctx,
11223 &mut module_rx,
11224 hello_frame("aft", PROTOCOL_VERSION, 7),
11225 )
11226 .await;
11227 let project_root = unique_project_root("role-versions");
11228 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
11229 let route_handler = handler.clone();
11230 let route_task = tokio::spawn(async move {
11231 route_handler
11232 .handle_control_frame(
11233 &client_ctx,
11234 route_open_frame_with_role_versions(403, "aft", project_root, sent),
11235 )
11236 .await
11237 .unwrap()
11238 });
11239 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11240 .await
11241 .expect("a well-formed route.open reaches the module as a bind")
11242 .unwrap();
11243 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11244 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
11245 panic!("expected route.bind request, got {bind:?}");
11246 };
11247 handler
11248 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11249 .await
11250 .unwrap();
11251 assert!(route_task.await.unwrap().is_empty());
11252 let published = client_rx.recv().await.unwrap();
11253 assert!(matches!(
11254 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11255 ClientControlResponse::RouteOpen { .. }
11256 ));
11257 role_versions
11258 }
11259
11260 #[tokio::test]
11261 async fn route_open_relays_role_versions_verbatim() {
11262 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
11263 assert_eq!(
11264 bind_role_versions_for(Some(sent.clone()), 141).await,
11265 Some(sent)
11266 );
11267 }
11268
11269 #[tokio::test]
11272 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
11273 assert_eq!(bind_role_versions_for(None, 143).await, None);
11274 assert_eq!(
11275 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
11276 None
11277 );
11278 }
11279
11280 #[tokio::test]
11281 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
11282 let registry = Arc::new(Registry::default());
11283 let forwarding = Arc::new(ForwardingTable::default());
11284 let handler =
11285 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11286 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
11287 hello_via_sink(
11288 &handler,
11289 &module_ctx,
11290 &mut module_rx,
11291 hello_frame("aft", PROTOCOL_VERSION, 7),
11292 )
11293 .await;
11294
11295 let nine: BTreeMap<String, String> = (0..9)
11296 .map(|index| (format!("role-{index}"), "v1".to_string()))
11297 .collect();
11298 for (label, malformed) in [
11299 (
11300 "invalid role name",
11301 role_versions(&[("Tool_Provider", "v1")]),
11302 ),
11303 ("invalid version", role_versions(&[("tool-provider", "v0")])),
11304 ("nine entries", nine),
11305 ] {
11306 let responses = tokio::time::timeout(
11309 Duration::from_secs(1),
11310 handler.handle_control_frame(
11311 &route_ctx(ConnectionId::new(148)).0,
11312 route_open_frame_with_role_versions(
11313 404,
11314 "aft",
11315 unique_project_root("role-versions-malformed"),
11316 Some(malformed),
11317 ),
11318 ),
11319 )
11320 .await
11321 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11322 .unwrap();
11323 assert_eq!(responses.len(), 1, "{label}");
11324 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11325 let error = parse_error(&responses[0]);
11326 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11327 assert_eq!(
11328 error["detail"]["field"], "role_versions",
11329 "{label}: {error}"
11330 );
11331 assert!(
11332 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11333 "{label}: a malformed declaration is terminal"
11334 );
11335 assert!(
11336 module_rx.try_recv().is_err(),
11337 "{label}: the module must never see a bind"
11338 );
11339 }
11340 }
11341
11342 #[tokio::test]
11346 async fn route_role_versions_capability_is_advertised() {
11347 let handler = ControlHandler::new(Arc::new(Registry::default()));
11348 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11349 let ack = hello_via_sink(
11350 &handler,
11351 &ctx,
11352 &mut rx,
11353 hello_frame("m", PROTOCOL_VERSION, 1),
11354 )
11355 .await;
11356 let ack = parse_ack(&ack);
11357 assert!(
11358 ack.subc_capabilities
11359 .iter()
11360 .any(|c| c == "route-role-versions/v1"),
11361 "{:?}",
11362 ack.subc_capabilities
11363 );
11364
11365 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11366 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11367 let reply = handler
11368 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11369 .await
11370 .unwrap()
11371 .pop()
11372 .unwrap();
11373 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11374 serde_json::from_slice(&reply.body).unwrap()
11375 else {
11376 panic!("not a server.describe reply");
11377 };
11378 assert!(
11379 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
11380 "{capabilities:?}"
11381 );
11382 }
11383
11384 #[tokio::test]
11388 async fn catalog_events_capability_is_advertised() {
11389 let handler = ControlHandler::new(Arc::new(Registry::default()));
11390 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11391 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11392 let reply = handler
11393 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
11394 .await
11395 .unwrap()
11396 .pop()
11397 .unwrap();
11398 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11399 serde_json::from_slice(&reply.body).unwrap()
11400 else {
11401 panic!("not a server.describe reply");
11402 };
11403 assert!(
11404 capabilities
11405 .iter()
11406 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
11407 "{capabilities:?}"
11408 );
11409 }
11410
11411 #[tokio::test]
11412 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
11413 let handler = ControlHandler::new(Arc::new(Registry::default()));
11414 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
11415 let ack = hello_via_sink(
11416 &handler,
11417 &module,
11418 &mut module_rx,
11419 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
11420 )
11421 .await;
11422 let ack = parse_ack(&ack);
11423 assert!(
11424 ack.subc_capabilities
11425 .iter()
11426 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11427 "{:?}",
11428 ack.subc_capabilities
11429 );
11430 assert!(ack
11431 .subc_capabilities
11432 .iter()
11433 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11434
11435 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
11436 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11437 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11438 let reply = handler
11439 .handle_control_frame(&client, frame)
11440 .await
11441 .unwrap()
11442 .pop()
11443 .unwrap();
11444 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11445 serde_json::from_slice(&reply.body).unwrap()
11446 else {
11447 panic!("not a server.describe reply");
11448 };
11449 assert!(
11450 capabilities
11451 .iter()
11452 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11453 "{capabilities:?}"
11454 );
11455 assert!(capabilities
11456 .iter()
11457 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11458 }
11459
11460 #[tokio::test]
11461 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
11462 let handler = ControlHandler::new(Arc::new(Registry::default()));
11463 let (module, mut rx) = route_ctx(ConnectionId::new(156));
11464 let ack = hello_via_sink(
11465 &handler,
11466 &module,
11467 &mut rx,
11468 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
11469 )
11470 .await;
11471 assert!(parse_ack(&ack)
11472 .subc_capabilities
11473 .iter()
11474 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
11475 assert!(parse_ack(&ack)
11476 .subc_capabilities
11477 .iter()
11478 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
11479 let (client, _rx) = route_ctx(ConnectionId::new(157));
11480 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11481 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11482 let reply = handler
11483 .handle_control_frame(&client, frame)
11484 .await
11485 .unwrap()
11486 .pop()
11487 .unwrap();
11488 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11489 serde_json::from_slice(&reply.body).unwrap()
11490 else {
11491 panic!("server.describe reply")
11492 };
11493 assert!(capabilities
11494 .iter()
11495 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
11496 assert!(capabilities
11497 .iter()
11498 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
11499 }
11500
11501 #[tokio::test]
11502 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
11503 let handler = ControlHandler::new(Arc::new(Registry::default()));
11504 let (module, mut rx) = route_ctx(ConnectionId::new(154));
11505 let ack = hello_via_sink(
11506 &handler,
11507 &module,
11508 &mut rx,
11509 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
11510 )
11511 .await;
11512 assert!(parse_ack(&ack)
11513 .subc_capabilities
11514 .iter()
11515 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11516 let (client, _rx) = route_ctx(ConnectionId::new(155));
11517 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11518 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11519 let reply = handler
11520 .handle_control_frame(&client, frame)
11521 .await
11522 .unwrap()
11523 .pop()
11524 .unwrap();
11525 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11526 serde_json::from_slice(&reply.body).unwrap()
11527 else {
11528 panic!("not a server.describe reply")
11529 };
11530 assert!(capabilities
11531 .iter()
11532 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11533 }
11534
11535 #[tokio::test]
11536 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
11537 let registry = Arc::new(Registry::default());
11538 let forwarding = Arc::new(ForwardingTable::default());
11539 let handler =
11540 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11541 .with_health_probe_timeout(Duration::from_secs(5));
11542 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
11543 hello_via_sink(
11544 &handler,
11545 &module_ctx,
11546 &mut module_rx,
11547 non_routable_hello_frame_with_control_ops(
11548 "mcp",
11549 300,
11550 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11551 ),
11552 )
11553 .await;
11554 assert!(registry
11555 .get_module("mcp")
11556 .unwrap()
11557 .unwrap()
11558 .manifest
11559 .provides
11560 .is_empty());
11561
11562 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
11563 let route_response = handler
11564 .handle_control_frame(
11565 &route_client_ctx,
11566 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
11567 )
11568 .await
11569 .unwrap();
11570 assert_eq!(route_response[0].header.ty, FrameType::Error);
11571 assert_eq!(
11572 parse_error(&route_response[0])["code"],
11573 "target_unavailable"
11574 );
11575 assert!(parse_error(&route_response[0])["message"]
11576 .as_str()
11577 .unwrap()
11578 .contains("does not provide the requested target"));
11579 assert!(module_rx.try_recv().is_err());
11580
11581 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
11582 let health_handler = handler.clone();
11583 let health_task = tokio::spawn(async move {
11584 health_handler
11585 .handle_control_frame(
11586 &health_client_ctx,
11587 supervisor_health_probe_frame(302, "mcp"),
11588 )
11589 .await
11590 .unwrap()
11591 });
11592 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11593 .await
11594 .unwrap()
11595 .unwrap();
11596 assert_eq!(
11597 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11598 ModuleControlRequest::HealthCheck {}
11599 );
11600 handler
11601 .handle_control_frame(
11602 &module_ctx,
11603 health_response(health_frame.header.corr, HealthStatus::Ok),
11604 )
11605 .await
11606 .unwrap();
11607 let health_response = health_task.await.unwrap();
11608 assert_eq!(health_response[0].header.ty, FrameType::Response);
11609 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11610 ClientControlResponse::SupervisorHealthProbe {
11611 module_id, status, ..
11612 } => {
11613 assert_eq!(module_id, "mcp");
11614 assert_eq!(status, HealthStatus::Ok);
11615 }
11616 other => panic!("unexpected health response: {other:?}"),
11617 }
11618
11619 forwarding
11624 .cleanup_connection(module_ctx.connection_id)
11625 .unwrap();
11626 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
11627 let cleanup_response = tokio::time::timeout(
11628 Duration::from_millis(200),
11629 handler.handle_control_frame(
11630 &cleanup_probe_ctx,
11631 supervisor_health_probe_frame(303, "mcp"),
11632 ),
11633 )
11634 .await
11635 .expect("probe should fail immediately when the control lane is gone")
11636 .unwrap();
11637 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
11638 assert_eq!(
11639 parse_error(&cleanup_response[0])["code"],
11640 "target_unavailable"
11641 );
11642 assert!(parse_error(&cleanup_response[0])["message"]
11643 .as_str()
11644 .unwrap()
11645 .contains("no module connection"));
11646
11647 handler
11648 .cleanup_connection(module_ctx.connection_id)
11649 .unwrap();
11650 }
11651
11652 #[tokio::test]
11653 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
11654 let registry = Arc::new(Registry::default());
11655 let supervisor_handle = SupervisorHandle::new();
11656 let supervisor =
11657 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11658 .with_handle(supervisor_handle.clone())
11659 .with_connection_file_path(
11660 std::env::temp_dir()
11661 .join(format!("subc-route-open-warming-{}", std::process::id())),
11662 );
11663 let module = supervisor
11664 .supervise_configured(
11665 ModuleSpec {
11666 module_id: "warming".to_string(),
11667 program: fake_aft_stub_path(),
11668 args: Vec::new(),
11669 env: Vec::new(),
11670 reserved: false,
11671 reserved_prefixes: Vec::new(),
11672 protocol: ModuleProtocol::Subc,
11673 overlap: Default::default(),
11674 },
11675 true,
11676 )
11677 .unwrap();
11678 assert_eq!(module.state().unwrap(), ModuleState::Running);
11679
11680 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11681 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
11682 let response = handler
11683 .handle_control_frame(
11684 &ctx,
11685 route_open_frame(304, "warming", unique_project_root("warming")),
11686 )
11687 .await
11688 .unwrap();
11689 module.stop().await.unwrap();
11690
11691 assert_eq!(response[0].header.ty, FrameType::Error);
11692 let error = parse_error(&response[0]);
11693 assert_eq!(error["code"], "module_warming");
11694 assert!(error["message"]
11695 .as_str()
11696 .unwrap()
11697 .contains("state=running, enabled=true, live=false"));
11698 }
11699
11700 #[test]
11701 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
11702 let handler = ControlHandler::new(Arc::new(Registry::default()));
11703 let capture = EventCapture::default();
11704 let _subscriber =
11705 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11706 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
11707 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
11708 let pending = (0..limit).collect::<Vec<_>>();
11709 let response = handler
11710 .route_open_capacity_refusal(
11711 &ctx,
11712 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
11713 "busy",
11714 pending.len(),
11715 limit,
11716 )
11717 .unwrap();
11718 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11719 let event = capture
11720 .events()
11721 .into_iter()
11722 .find(|event| {
11723 event.target == "control"
11724 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
11725 })
11726 .expect("connection admission refusal event");
11727 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11728 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11729 }
11730
11731 #[test]
11732 fn route_open_target_cap_logs_admission_reason_and_capacity() {
11733 let handler = ControlHandler::new(Arc::new(Registry::default()));
11734 let capture = EventCapture::default();
11735 let _subscriber =
11736 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11737 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
11738 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
11739 let guards = (0..limit)
11740 .map(|_| {
11741 handler
11742 .route_bind_concurrency
11743 .try_admit("busy", limit)
11744 .unwrap()
11745 })
11746 .collect::<Vec<_>>();
11747 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
11748 Err(in_flight) => in_flight,
11749 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
11750 };
11751 let response = handler
11752 .route_open_target_capacity_refusal(
11753 &ctx,
11754 &route_open_frame(397, "busy", unique_project_root("target-cap")),
11755 "busy",
11756 in_flight,
11757 )
11758 .unwrap();
11759 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11760 let event = capture
11761 .events()
11762 .into_iter()
11763 .find(|event| {
11764 event.target == "control"
11765 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
11766 })
11767 .expect("target admission refusal event");
11768 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11769 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11770 drop(guards);
11771 }
11772
11773 #[tokio::test(flavor = "current_thread")]
11774 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
11775 let handler = ControlHandler::new(Arc::new(Registry::default()));
11776 let capture = EventCapture::default();
11777 let _subscriber =
11778 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11779 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
11780 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11781
11782 let _ = handler
11783 .handle_client_control_request(
11784 &ctx,
11785 frame,
11786 ClientControlRequest::SupervisorSetEnabled {
11787 module_id: "broca".to_string(),
11788 enabled: false,
11789 },
11790 &mut None,
11791 )
11792 .await
11793 .unwrap();
11794
11795 let events = capture
11796 .events()
11797 .into_iter()
11798 .filter(|event| {
11799 event.target == "control"
11800 && event.fields.get("message").map(String::as_str)
11801 == Some("supervisor request received")
11802 })
11803 .collect::<Vec<_>>();
11804 assert_eq!(events.len(), 1, "one supervisor request log line");
11805 let event = &events[0];
11806 assert_eq!(event.level, tracing::Level::INFO);
11807 assert_eq!(
11808 event.fields.get("op"),
11809 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
11810 );
11811 assert_eq!(
11812 event.fields.get("module_id"),
11813 Some(&"\"broca\"".to_string())
11814 );
11815 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
11816 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
11817 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
11818 }
11819
11820 #[tokio::test(flavor = "current_thread")]
11821 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
11822 let registry = Arc::new(Registry::default());
11823 let module_connection = ConnectionId::new(303);
11824 registry
11825 .register_with_control_ops(
11826 manifest("aft", PROTOCOL_VERSION),
11827 PROTOCOL_VERSION,
11828 module_connection,
11829 module_baseline_control_ops(),
11830 )
11831 .unwrap();
11832 let handler = ControlHandler::new(registry);
11833 let capture = EventCapture::default();
11834 let _subscriber =
11835 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11836 let (ctx, _rx) = route_ctx(module_connection);
11837 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11838
11839 let _ = handler
11840 .handle_client_control_request(
11841 &ctx,
11842 frame,
11843 ClientControlRequest::SupervisorSetEnabled {
11844 module_id: "broca".to_string(),
11845 enabled: false,
11846 },
11847 &mut None,
11848 )
11849 .await
11850 .unwrap();
11851
11852 let callers = capture
11853 .events()
11854 .into_iter()
11855 .filter(|event| {
11856 event.fields.get("message").map(String::as_str)
11857 == Some("supervisor request received")
11858 })
11859 .map(|event| event.fields.get("caller").cloned())
11860 .collect::<Vec<_>>();
11861 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
11862 }
11863
11864 #[tokio::test(flavor = "current_thread")]
11865 async fn supervisor_list_does_not_log_request_received() {
11866 let handler = ControlHandler::new(Arc::new(Registry::default()));
11867 let capture = EventCapture::default();
11868 let _subscriber =
11869 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11870 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
11871 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
11872
11873 handler
11874 .handle_client_control_request(
11875 &ctx,
11876 frame,
11877 ClientControlRequest::SupervisorList {},
11878 &mut None,
11879 )
11880 .await
11881 .unwrap();
11882
11883 assert!(capture.events().into_iter().all(|event| {
11884 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11885 }));
11886 }
11887
11888 #[tokio::test(flavor = "current_thread")]
11889 async fn supervisor_rescan_preview_does_not_log_request_received() {
11890 let handler = ControlHandler::new(Arc::new(Registry::default()));
11891 let capture = EventCapture::default();
11892 let _subscriber =
11893 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11894 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
11895 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
11896
11897 handler
11898 .handle_client_control_request(
11899 &ctx,
11900 frame,
11901 ClientControlRequest::SupervisorRescan { preview: true },
11902 &mut None,
11903 )
11904 .await
11905 .unwrap();
11906
11907 assert!(capture.events().into_iter().all(|event| {
11908 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11909 }));
11910 }
11911
11912 struct ApproveOperator;
11913 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
11914 fn prompt(
11915 &self,
11916 _: &str,
11917 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
11918 ) -> crate::operator_confirm::ProviderResult {
11919 crate::operator_confirm::ProviderResult::Approved
11920 }
11921 }
11922
11923 #[tokio::test]
11924 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
11925 assert_eq!(
11926 ControlHandler::default().privileged_operator_prompt,
11927 PrivilegedOperatorPrompt::Log
11928 );
11929 for (initial, changed, running) in [
11930 ("log", "enforce", PrivilegedOperatorPrompt::Log),
11931 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
11932 ] {
11933 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
11934 let path = dir.join("subc.jsonc");
11935 std::fs::write(
11936 &path,
11937 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
11938 )
11939 .unwrap();
11940 let config = crate::daemon_config::load(&path).unwrap().unwrap();
11941 let registry = Arc::new(Registry::default());
11942 let handle = SupervisorHandle::new();
11943 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
11944 .with_handle(handle.clone());
11945 let handler = ControlHandler::new(registry)
11946 .with_supervisor(handle)
11947 .with_privileged_operator_prompt(config.privileged_operator_prompt)
11948 .with_supervisor_rescan(supervisor, &path, config.port);
11949 handler.forwarding.operator_confirms().configure(
11950 Arc::new(ApproveOperator),
11951 crate::operator_confirm::OperatorLimits::default(),
11952 );
11953 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
11954
11955 for (disk_mode, needs_restart) in [
11956 (Some(initial), false),
11957 (Some(changed), true),
11958 (Some(changed), true),
11959 (None, running == PrivilegedOperatorPrompt::Enforce),
11960 (Some(initial), false),
11961 ] {
11962 let mut doc = json!({"version":1});
11963 if let Some(mode) = disk_mode {
11964 doc["privileged_operator_prompt"] = json!(mode);
11965 }
11966 std::fs::write(&path, doc.to_string()).unwrap();
11967 for preview in [true, false] {
11968 let request = ClientControlRequest::SupervisorRescan { preview };
11969 let frame = Frame::build(
11970 FrameType::Request,
11971 control_flags(),
11972 0,
11973 0,
11974 1,
11975 serde_json::to_vec(&request).unwrap(),
11976 )
11977 .unwrap();
11978 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11979 assert_eq!(frames.len(), 1);
11980 let ClientControlResponse::SupervisorRescan { result } =
11981 serde_json::from_slice(&frames[0].body).unwrap()
11982 else {
11983 panic!("expected a rescan response");
11984 };
11985 let expected = if needs_restart {
11986 vec!["privileged_operator_prompt"]
11987 } else {
11988 Vec::new()
11989 };
11990 assert_eq!(result.restart_required, expected);
11991 assert_eq!(handler.privileged_operator_prompt, running);
11992 assert_eq!(
11993 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
11994 running
11995 );
11996 }
11997 }
11998 }
11999 }
12000
12001 #[tokio::test]
12002 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
12003 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
12004 let path = dir.join("subc.jsonc");
12005 let registry = Arc::new(Registry::default());
12006 let handle = SupervisorHandle::new();
12007 let supervisor =
12008 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
12009 let handler = ControlHandler::new(registry)
12010 .with_supervisor(handle)
12011 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
12012 .with_supervisor_rescan(supervisor, &path, None);
12013 handler.forwarding.operator_confirms().configure(
12014 Arc::new(ApproveOperator),
12015 crate::operator_confirm::OperatorLimits::default(),
12016 );
12017 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
12018 for preview in [true, false] {
12019 std::fs::write(
12020 &path,
12021 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
12022 )
12023 .unwrap();
12024 let request = ClientControlRequest::SupervisorRescan { preview };
12025 let frame = Frame::build(
12026 FrameType::Request,
12027 control_flags(),
12028 0,
12029 0,
12030 1,
12031 serde_json::to_vec(&request).unwrap(),
12032 )
12033 .unwrap();
12034 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12035 assert_eq!(frames.len(), 1);
12036 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
12037 assert_eq!(
12038 handler.privileged_operator_prompt,
12039 PrivilegedOperatorPrompt::Enforce
12040 );
12041 }
12042 }
12043
12044 #[tokio::test]
12049 async fn route_open_refusal_names_the_check_that_refused() {
12050 let handler = ControlHandler::new(Arc::new(Registry::default()));
12051 let capture = EventCapture::default();
12052 let _subscriber =
12053 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12054 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12055 let response = handler
12056 .handle_control_frame(
12057 &ctx,
12058 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
12059 )
12060 .await
12061 .unwrap();
12062
12063 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12064 let event = capture
12065 .events()
12066 .into_iter()
12067 .find(|event| {
12068 event.target == "control"
12069 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12070 })
12071 .expect("route.open refusal event");
12072 assert_eq!(
12073 event.fields.get("reason"),
12074 Some(&"\"not_registered\"".to_string())
12075 );
12076 }
12077
12078 #[tokio::test]
12079 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
12080 let registry = Arc::new(Registry::default());
12081 let supervisor_handle = SupervisorHandle::new();
12082 let supervisor =
12083 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12084 .with_handle(supervisor_handle.clone())
12085 .with_connection_file_path(std::env::temp_dir().join(format!(
12086 "subc-route-open-refusal-info-{}",
12087 std::process::id()
12088 )));
12089 let module = supervisor
12090 .supervise_configured(
12091 ModuleSpec {
12092 module_id: "warming".to_string(),
12093 program: fake_aft_stub_path(),
12094 args: Vec::new(),
12095 env: Vec::new(),
12096 reserved: false,
12097 reserved_prefixes: Vec::new(),
12098 protocol: ModuleProtocol::Subc,
12099 overlap: Default::default(),
12100 },
12101 true,
12102 )
12103 .unwrap();
12104 assert_eq!(module.state().unwrap(), ModuleState::Running);
12105
12106 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12107 assert!(handler
12108 .counters()
12109 .snapshot()
12110 .get("route_open_refused_by_code")
12111 .is_none());
12112 let capture = EventCapture::default();
12113 let _subscriber =
12114 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12115 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
12116 let response = handler
12117 .handle_control_frame(
12118 &ctx,
12119 route_open_frame(394, "warming", unique_project_root("refusal-info")),
12120 )
12121 .await
12122 .unwrap();
12123 module.stop().await.unwrap();
12124
12125 assert_eq!(parse_error(&response[0])["code"], "module_warming");
12126 let event = capture
12127 .events()
12128 .into_iter()
12129 .find(|event| {
12130 event.target == "control"
12131 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
12132 })
12133 .expect("route.open refusal event");
12134 assert_eq!(
12135 event.fields.get("module_id"),
12136 Some(&"\"warming\"".to_string())
12137 );
12138 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
12139 assert_eq!(
12140 event.fields.get("reason"),
12141 Some(&"\"supervised_not_registered\"".to_string())
12142 );
12143 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
12144 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
12145 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
12146 assert_eq!(
12147 handler.counters().snapshot()["route_open_refused_by_code"],
12148 json!({ "module_warming": 1 })
12149 );
12150 }
12151
12152 const OUTAGE_START: &str = "route.open refusing module: not serving";
12153 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
12154
12155 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
12156 capture
12157 .events()
12158 .into_iter()
12159 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
12160 .collect()
12161 }
12162
12163 fn supervise_stub(
12164 registry: &Arc<Registry>,
12165 module_id: &str,
12166 enabled: bool,
12167 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
12168 let supervisor_handle = SupervisorHandle::new();
12169 let supervisor =
12170 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
12171 .with_handle(supervisor_handle.clone())
12172 .with_connection_file_path(std::env::temp_dir().join(format!(
12173 "subc-route-outage-{module_id}-{}",
12174 std::process::id()
12175 )));
12176 let module = supervisor
12177 .supervise_configured(
12178 ModuleSpec {
12179 module_id: module_id.to_string(),
12180 program: fake_aft_stub_path(),
12181 args: Vec::new(),
12182 env: Vec::new(),
12183 reserved: false,
12184 reserved_prefixes: Vec::new(),
12185 protocol: ModuleProtocol::Subc,
12186 overlap: Default::default(),
12187 },
12188 enabled,
12189 )
12190 .unwrap();
12191 (supervisor_handle, module)
12192 }
12193
12194 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
12195 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
12196 module_id: module_id.to_string(),
12197 drain_timeout_ms: Some(50),
12198 })
12199 .unwrap();
12200 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
12201 }
12202
12203 #[test]
12207 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
12208 let registry = Arc::new(Registry::default());
12209 let forwarding = Arc::new(ForwardingTable::default());
12210 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12211 let second = ControlHandler::with_forwarding(registry, forwarding);
12212 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
12213 }
12214
12215 #[tokio::test(flavor = "current_thread")]
12219 async fn route_open_unknown_module_refusals_add_no_outage_state() {
12220 let handler = ControlHandler::new(Arc::new(Registry::default()));
12221 let capture = EventCapture::default();
12222 let _subscriber =
12223 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12224 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
12225 for corr in 0..8 {
12226 let response = handler
12227 .handle_control_frame(
12228 &ctx,
12229 route_open_frame(
12230 380 + corr,
12231 &format!("nobody-{corr}"),
12232 unique_project_root("outage-unknown"),
12233 ),
12234 )
12235 .await
12236 .unwrap();
12237 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12238 }
12239
12240 assert_eq!(handler.route_outages.tracked_module_count(), 0);
12241 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
12242 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
12243 }
12244
12245 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12250 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
12251 let registry = Arc::new(Registry::default());
12252 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
12253 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12254 let capture = EventCapture::default();
12255 let _subscriber =
12256 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12257 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
12258 handler.route_outages.record_accepted("outage-restart");
12261
12262 let response = handler
12263 .handle_control_frame(
12264 &ctx,
12265 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
12266 )
12267 .await
12268 .unwrap();
12269 assert_eq!(response[0].header.ty, FrameType::Error);
12270 let starts = outage_lines(&capture, OUTAGE_START);
12271 assert_eq!(starts.len(), 1, "{starts:?}");
12272 assert_eq!(starts[0].level, tracing::Level::WARN);
12273 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
12274 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
12275 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
12276 handler.route_outages.record_accepted("outage-restart");
12277 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
12278
12279 let restart = handler
12280 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
12281 .await
12282 .unwrap();
12283 assert_eq!(
12284 restart[0].header.ty,
12285 FrameType::Response,
12286 "{:?}",
12287 parse_error(&restart[0])
12288 );
12289 handler
12290 .handle_control_frame(
12291 &ctx,
12292 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
12293 )
12294 .await
12295 .unwrap();
12296 module.stop().await.unwrap();
12297
12298 let starts = outage_lines(&capture, OUTAGE_START);
12299 assert_eq!(starts.len(), 2, "{starts:?}");
12300 assert_eq!(starts[1].level, tracing::Level::INFO);
12301 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
12302 }
12303
12304 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12308 async fn failed_operator_restart_leaves_no_operator_mark() {
12309 let registry = Arc::new(Registry::default());
12310 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
12311 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12312 let capture = EventCapture::default();
12313 let _subscriber =
12314 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12315 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
12316 handler.route_outages.record_accepted("outage-disabled");
12317
12318 let restart = handler
12319 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
12320 .await
12321 .unwrap();
12322 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
12323 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
12324
12325 handler
12326 .handle_control_frame(
12327 &ctx,
12328 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
12329 )
12330 .await
12331 .unwrap();
12332 let starts = outage_lines(&capture, OUTAGE_START);
12333 assert_eq!(starts.len(), 1, "{starts:?}");
12334 assert_eq!(starts[0].level, tracing::Level::WARN);
12335 }
12336
12337 #[tokio::test(flavor = "current_thread")]
12338 async fn route_open_unknown_module_escapes_target_module_id() {
12339 let handler = ControlHandler::new(Arc::new(Registry::default()));
12340 let capture = EventCapture::default();
12341 let _subscriber =
12342 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12343 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
12344 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12345 let response = handler
12346 .handle_control_frame(
12347 &ctx,
12348 route_open_frame(
12349 395,
12350 hostile_module_id,
12351 unique_project_root("hostile-target-module-id"),
12352 ),
12353 )
12354 .await
12355 .unwrap();
12356
12357 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12358 let event = capture
12359 .events()
12360 .into_iter()
12361 .find(|event| {
12362 event.target == "control"
12363 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12364 })
12365 .expect("route.open unknown-module refusal event");
12366 let logged = event.fields.get("module_id").expect("module_id field");
12367 assert!(!logged.bytes().any(|byte| byte < 0x20));
12368 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12369 }
12370
12371 #[tokio::test(flavor = "current_thread")]
12372 async fn route_open_module_rejection_uses_daemon_counter_key() {
12373 let registry = Arc::new(Registry::default());
12374 let forwarding = Arc::new(ForwardingTable::default());
12375 let handler =
12376 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12377 let module_connection = ConnectionId::new(95);
12378 let (module_ctx, mut module_rx) = route_ctx(module_connection);
12379 hello_via_sink(
12380 &handler,
12381 &module_ctx,
12382 &mut module_rx,
12383 hello_frame("aft", PROTOCOL_VERSION, 395),
12384 )
12385 .await;
12386
12387 let client_connection = ConnectionId::new(96);
12388 let (client_ctx, _client_rx) = route_ctx(client_connection);
12389 let capture = EventCapture::default();
12390 let _subscriber =
12391 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12392 let (route_task, bind) = relay_route_open(
12393 &handler,
12394 client_connection,
12395 &client_ctx.egress,
12396 &mut module_rx,
12397 396,
12398 "aft",
12399 "hostile-module-code",
12400 )
12401 .await;
12402 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
12403 let rejection = Frame::build(
12404 FrameType::Error,
12405 control_flags(),
12406 0,
12407 0,
12408 bind.header.corr,
12409 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
12410 )
12411 .unwrap();
12412 handler
12413 .handle_control_frame(&module_ctx, rejection)
12414 .await
12415 .unwrap();
12416
12417 let response = route_task.await.unwrap();
12418 assert_eq!(parse_error(&response[0])["code"], hostile_code);
12419 let counters = handler.counters().snapshot();
12420 assert_eq!(
12421 counters["route_open_refused_by_code"],
12422 json!({ "module_rejected": 1 })
12423 );
12424 assert!(counters["route_open_refused_by_code"]
12425 .get(hostile_code)
12426 .is_none());
12427
12428 let event = capture
12429 .events()
12430 .into_iter()
12431 .find(|event| {
12432 event.target == "control"
12433 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
12434 })
12435 .expect("route.open module-rejection refusal event");
12436 let logged = event.fields.get("module_code").expect("module_code field");
12437 assert!(!logged.bytes().any(|byte| byte < 0x20));
12438 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12439 }
12440
12441 #[tokio::test]
12442 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
12443 let registry = Arc::new(Registry::default());
12444 let supervisor_handle = SupervisorHandle::new();
12445 let missing_program = std::env::temp_dir().join(format!(
12446 "subc-route-open-missing-program-{}",
12447 std::process::id()
12448 ));
12449 let supervisor =
12450 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12451 .with_handle(supervisor_handle.clone());
12452 let module = supervisor
12453 .supervise_configured(
12454 ModuleSpec {
12455 module_id: "failed".to_string(),
12456 program: missing_program,
12457 args: Vec::new(),
12458 env: Vec::new(),
12459 reserved: false,
12460 reserved_prefixes: Vec::new(),
12461 protocol: ModuleProtocol::Subc,
12462 overlap: Default::default(),
12463 },
12464 true,
12465 )
12466 .unwrap();
12467 assert_eq!(module.state().unwrap(), ModuleState::Failed);
12468
12469 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12470 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
12471 let response = handler
12472 .handle_control_frame(
12473 &ctx,
12474 route_open_frame(305, "failed", unique_project_root("failed")),
12475 )
12476 .await
12477 .unwrap();
12478
12479 assert_eq!(response[0].header.ty, FrameType::Error);
12480 let error = parse_error(&response[0]);
12481 assert_eq!(error["code"], "target_unavailable");
12482 assert!(error["message"]
12483 .as_str()
12484 .unwrap()
12485 .contains("state=failed, enabled=true, live=false"));
12486 }
12487
12488 #[tokio::test]
12489 async fn route_open_role_mismatch_remains_target_unavailable() {
12490 let registry = Arc::new(Registry::default());
12491 let handler = ControlHandler::new(Arc::clone(®istry));
12492 handler
12493 .handle_control(
12494 ConnectionId::new(41),
12495 non_routable_hello_frame_with_control_ops("health-only", 306, None),
12496 )
12497 .unwrap();
12498
12499 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
12500 let response = handler
12501 .handle_control_frame(
12502 &ctx,
12503 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
12504 )
12505 .await
12506 .unwrap();
12507
12508 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12509 assert!(parse_error(&response[0])["message"]
12510 .as_str()
12511 .unwrap()
12512 .contains("does not provide the requested target"));
12513 }
12514
12515 #[tokio::test]
12516 async fn route_open_inactive_registration_remains_target_unavailable() {
12517 let registry = Arc::new(Registry::default());
12518 let handler = ControlHandler::new(Arc::clone(®istry));
12519 handler
12520 .handle_control(
12521 ConnectionId::new(43),
12522 hello_frame("inactive", PROTOCOL_VERSION, 308),
12523 )
12524 .unwrap();
12525 assert!(registry
12526 .set_module_state_for_test("inactive", ChannelState::Closed)
12527 .unwrap());
12528
12529 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
12530 let response = handler
12531 .handle_control_frame(
12532 &ctx,
12533 route_open_frame(309, "inactive", unique_project_root("inactive")),
12534 )
12535 .await
12536 .unwrap();
12537
12538 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12539 assert!(parse_error(&response[0])["message"]
12540 .as_str()
12541 .unwrap()
12542 .contains("is not active"));
12543 }
12544
12545 #[tokio::test]
12546 async fn late_health_reply_is_recorded_through_the_module_response_path() {
12547 let registry = Arc::new(Registry::default());
12548 let forwarding = Arc::new(ForwardingTable::default());
12549 let supervisor_handle = SupervisorHandle::new();
12550 let supervisor =
12551 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
12552 .with_forwarding(Arc::clone(&forwarding))
12553 .with_handle(supervisor_handle.clone());
12554 let module = supervisor
12555 .supervise_configured(
12556 crate::ModuleSpec {
12557 module_id: "late-health-response".to_string(),
12558 program: PathBuf::from("disabled-module"),
12559 args: Vec::new(),
12560 env: Vec::new(),
12561 reserved: false,
12562 reserved_prefixes: Vec::new(),
12563 protocol: ModuleProtocol::Subc,
12564 overlap: Default::default(),
12565 },
12566 false,
12567 )
12568 .unwrap();
12569 let handler =
12570 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12571 .with_supervisor(supervisor_handle);
12572 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
12573 handler
12574 .handle_control_frame(
12575 &module_ctx,
12576 hello_frame_with_control_ops(
12577 "late-health-response",
12578 PROTOCOL_VERSION,
12579 7,
12580 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12581 ),
12582 )
12583 .await
12584 .unwrap();
12585 let probe_started_at = Instant::now() - Duration::from_millis(80);
12586 let pending = forwarding
12587 .begin_health_probe_rpc_for(
12588 "late-health-response",
12589 MODULE_CONTROL_OP_HEALTH_CHECK,
12590 probe_started_at,
12591 Instant::now() - Duration::from_millis(1),
12592 )
12593 .unwrap();
12594 assert!(forwarding
12595 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
12596 .unwrap());
12597
12598 let responses = handler
12599 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
12600 .await
12601 .unwrap();
12602
12603 assert!(responses.is_empty());
12604 let health = module.status().unwrap().health;
12605 assert_eq!(health.late_answer_count, 1);
12606 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
12607 }
12608
12609 #[tokio::test]
12610 async fn health_probe_timeout_and_module_death_are_typed() {
12611 let registry = Arc::new(Registry::default());
12612 let forwarding = Arc::new(ForwardingTable::default());
12613 let handler =
12614 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12615 .with_health_probe_timeout(Duration::from_millis(50));
12616 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12617 hello_via_sink(
12618 &handler,
12619 &module_ctx,
12620 &mut module_rx,
12621 hello_frame_with_control_ops(
12622 "aft",
12623 PROTOCOL_VERSION,
12624 7,
12625 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12626 ),
12627 )
12628 .await;
12629
12630 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12631 let responses = handler
12632 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
12633 .await
12634 .unwrap();
12635 assert_eq!(responses[0].header.ty, FrameType::Error);
12636 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
12637 let _ = module_rx.try_recv();
12638
12639 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
12640 let health_handler = handler.clone();
12641 let death_task = tokio::spawn(async move {
12642 health_handler
12643 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
12644 .await
12645 .unwrap()
12646 });
12647 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12648 .await
12649 .unwrap()
12650 .unwrap();
12651 handler
12652 .cleanup_connection(module_ctx.connection_id)
12653 .unwrap();
12654 let responses = death_task.await.unwrap();
12655 assert_eq!(responses[0].header.ty, FrameType::Error);
12656 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
12657 }
12658
12659 #[test]
12660 fn hello_requires_exact_protocol_version() {
12661 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
12662 let registry = Arc::new(Registry::default());
12663 let handler = ControlHandler::new(Arc::clone(®istry));
12664 let responses = handler
12665 .handle_control(
12666 ConnectionId::new(connection),
12667 hello_frame("aft", offered, 9),
12668 )
12669 .unwrap();
12670
12671 assert_eq!(responses.len(), 1);
12672 assert_eq!(responses[0].header.ty, FrameType::Error);
12673 let error = parse_error(&responses[0]);
12674 assert_eq!(error["code"], "version_unsupported");
12675 assert!(registry.get_module("aft").unwrap().is_none());
12676 assert_eq!(registry.active_registration_count().unwrap(), 0);
12677 }
12678 }
12679
12680 #[test]
12681 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
12682 let registry = Arc::new(Registry::default());
12683 let forwarding = Arc::new(ForwardingTable::default());
12684 let handler =
12685 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12686 let module_connection = ConnectionId::new(301);
12687 let registration = registry
12688 .register_with_control_ops(
12689 manifest("aft-push", PROTOCOL_VERSION),
12690 PROTOCOL_VERSION,
12691 module_connection,
12692 module_baseline_control_ops(),
12693 )
12694 .unwrap();
12695 let (module_tx, _module_rx) = mpsc::channel(8);
12696 let endpoint = forwarding
12697 .register_module_connection(
12698 module_connection,
12699 "aft-push".to_string(),
12700 PROTOCOL_VERSION,
12701 manifest_concurrency(®istration.manifest),
12702 FrameSink::new(module_tx),
12703 )
12704 .unwrap();
12705
12706 let unknown = Frame::build(
12708 FrameType::Push,
12709 control_flags(),
12710 0,
12711 0,
12712 5,
12713 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
12714 )
12715 .unwrap();
12716 let out = handler.handle_status_update(endpoint, unknown).unwrap();
12717 assert!(
12718 out.is_empty(),
12719 "unknown push op must be ignored, got {out:?}"
12720 );
12721
12722 let malformed = Frame::build(
12724 FrameType::Push,
12725 control_flags(),
12726 0,
12727 0,
12728 6,
12729 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
12730 )
12731 .unwrap();
12732 let out = handler.handle_status_update(endpoint, malformed).unwrap();
12733 assert_eq!(out.len(), 1);
12734 assert_eq!(out[0].header.ty, FrameType::Error);
12735 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
12736 }
12737
12738 #[test]
12739 fn hello_rejected_when_connection_already_owns_client_routes() {
12740 let registry = Arc::new(Registry::default());
12741 let forwarding = Arc::new(ForwardingTable::default());
12742 let handler =
12743 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12744 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
12746 let client_connection = ConnectionId::new(202);
12747
12748 let responses = handler
12751 .handle_control(
12752 client_connection,
12753 hello_frame("aft-second", PROTOCOL_VERSION, 9),
12754 )
12755 .unwrap();
12756 assert_eq!(responses[0].header.ty, FrameType::Error);
12757 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
12758 assert!(registry.get_module("aft-second").unwrap().is_none());
12759 }
12760
12761 #[tokio::test]
12762 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
12763 let registry = Arc::new(Registry::default());
12764 let forwarding = Arc::new(ForwardingTable::default());
12765 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
12766 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
12767 hello_via_sink(
12768 &handler,
12769 &module_ctx,
12770 &mut module_rx,
12771 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
12772 )
12773 .await;
12774 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
12775 let pending = forwarding
12776 .begin_route_bind_relay_for_test(
12777 client_ctx.connection_id,
12778 client_ctx.egress.clone(),
12779 2,
12780 "alpha",
12781 )
12782 .unwrap();
12783 forwarding
12784 .complete_pending_relay(
12785 module_ctx.connection_id,
12786 pending.corr,
12787 RouteBindRelayOutcome::Accepted,
12788 )
12789 .unwrap();
12790 client_rx.try_recv().unwrap();
12791 for module_id in ["beta", "alpha"] {
12792 let replies = handler
12793 .handle_control_frame(
12794 &module_ctx,
12795 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
12796 )
12797 .await
12798 .unwrap();
12799 assert_eq!(replies.len(), 1, "second HELLO must be refused");
12800 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
12801 }
12802 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
12803 assert!(registry.get_module("beta").unwrap().is_none());
12804 assert!(matches!(
12805 forwarding
12806 .lookup_data_route(
12807 client_ctx.connection_id,
12808 pending.client_channel,
12809 pending.client_epoch,
12810 )
12811 .unwrap(),
12812 DataRoute::Client(DataRouteState::Bound(_))
12813 ));
12814 assert!(handler
12815 .hello_launch_nonces
12816 .lock()
12817 .unwrap()
12818 .presented(module_ctx.connection_id, Some("alpha-nonce")));
12819 assert!(module_rx.try_recv().is_err());
12820 }
12821
12822 #[test]
12823 fn reserved_module_hello_requires_matching_launch_nonce() {
12824 let registry = Arc::new(Registry::default());
12825 let supervisor = SupervisorHandle::new();
12826 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12829 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12830
12831 let no_nonce = handler
12833 .handle_control(
12834 ConnectionId::new(1),
12835 hello_frame("vault", PROTOCOL_VERSION, 1),
12836 )
12837 .unwrap();
12838 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
12839 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
12840 assert!(registry.get_module("vault").unwrap().is_none());
12841
12842 let wrong = handler
12844 .handle_control(
12845 ConnectionId::new(2),
12846 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
12847 )
12848 .unwrap();
12849 assert_eq!(wrong[0].header.ty, FrameType::Error);
12850 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
12851 assert!(registry.get_module("vault").unwrap().is_none());
12852
12853 let ok = handler
12855 .handle_control(
12856 ConnectionId::new(3),
12857 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
12858 )
12859 .unwrap();
12860 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
12861 assert!(registry.get_module("vault").unwrap().is_some());
12862 }
12863
12864 #[test]
12865 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
12866 let registry = Arc::new(Registry::default());
12867 let supervisor = SupervisorHandle::new();
12868 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12869 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12870 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12871
12872 let squat = handler
12873 .handle_control(
12874 ConnectionId::new(1),
12875 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
12876 )
12877 .unwrap();
12878 assert_eq!(squat[0].header.ty, FrameType::Error);
12879 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
12880 assert!(parse_error(&squat[0])["message"]
12881 .as_str()
12882 .unwrap()
12883 .contains("fed:"));
12884
12885 let accepted_peer = handler
12886 .handle_control(
12887 ConnectionId::new(2),
12888 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
12889 )
12890 .unwrap();
12891 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
12892
12893 let accepted_short = handler
12894 .handle_control(
12895 ConnectionId::new(3),
12896 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
12897 )
12898 .unwrap();
12899 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
12900
12901 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
12902 let response = handler
12903 .handle_control(
12904 ConnectionId::new(conn),
12905 hello_frame(module_id, PROTOCOL_VERSION, conn),
12906 )
12907 .unwrap();
12908 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
12909 }
12910 }
12911
12912 #[test]
12913 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
12914 let registry = Arc::new(Registry::default());
12915 let supervisor = SupervisorHandle::new();
12916 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12917 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12918 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
12919 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12920
12921 let owner_nonce = handler
12922 .handle_control(
12923 ConnectionId::new(1),
12924 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
12925 )
12926 .unwrap();
12927 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
12928 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
12929 assert!(registry.get_module("fed:special").unwrap().is_none());
12930
12931 let exact_nonce = handler
12932 .handle_control(
12933 ConnectionId::new(2),
12934 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
12935 )
12936 .unwrap();
12937 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
12938 assert!(registry.get_module("fed:special").unwrap().is_some());
12939 }
12940
12941 #[test]
12942 fn non_reserved_module_ignores_launch_nonce() {
12943 let registry = Arc::new(Registry::default());
12944 let handler = ControlHandler::new(Arc::clone(®istry));
12947 let no_nonce = handler
12948 .handle_control(
12949 ConnectionId::new(1),
12950 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
12951 )
12952 .unwrap();
12953 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
12954 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
12955
12956 let echoed_nonce = handler
12957 .handle_control(
12958 ConnectionId::new(2),
12959 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
12960 )
12961 .unwrap();
12962 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
12963 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
12964 }
12965
12966 #[test]
12967 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
12968 let handler = ControlHandler::default();
12969 let conn = ConnectionId::new(1);
12970 let malformed = Frame::build(
12971 FrameType::Hello,
12972 control_flags(),
12973 0,
12974 0,
12975 3,
12976 b"{not json".to_vec(),
12977 )
12978 .unwrap();
12979
12980 let error = handler.handle_control(conn, malformed).unwrap();
12981 assert_eq!(error[0].header.ty, FrameType::Error);
12982 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
12983
12984 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12985 let pong = handler.handle_control(conn, ping).unwrap();
12986 assert_eq!(pong[0].header.ty, FrameType::Pong);
12987 assert_eq!(pong[0].header.corr, 4);
12988 }
12989
12990 #[test]
12991 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
12992 let registry = Arc::new(Registry::default());
12993 let handler = ControlHandler::new(Arc::clone(®istry));
12994
12995 handler
12996 .handle_control(
12997 ConnectionId::new(1),
12998 hello_frame("aft", PROTOCOL_VERSION, 1),
12999 )
13000 .unwrap();
13001 let duplicate = handler
13002 .handle_control(
13003 ConnectionId::new(2),
13004 hello_frame("aft", PROTOCOL_VERSION, 2),
13005 )
13006 .unwrap();
13007
13008 assert_eq!(duplicate[0].header.ty, FrameType::Error);
13009 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13010 let registration = registry.get_module("aft").unwrap().unwrap();
13011 assert_eq!(registration.connection_id, ConnectionId::new(1));
13012 }
13013
13014 #[test]
13015 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
13016 let registry = Arc::new(Registry::default());
13017 let forwarding = Arc::new(ForwardingTable::default());
13018 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
13019 let handler =
13020 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13021 .with_process_liveness(process_liveness);
13022 let (ctx, route_channel, route_epoch) =
13023 bind_liveness_route(®istry, &forwarding, "aft-dead");
13024 let responses = handler
13025 .handle_route_poll(
13026 &ctx,
13027 route_poll_frame(41, PollKind::Liveness, route_channel),
13028 route_channel,
13029 route_epoch,
13030 PollKind::Liveness,
13031 )
13032 .unwrap();
13033
13034 assert_eq!(responses.len(), 1);
13035 assert_eq!(responses[0].header.ty, FrameType::Response);
13036 assert_route_poll_liveness(&responses[0], false);
13037 }
13038
13039 #[test]
13040 fn liveness_poll_without_process_source_uses_bound_route() {
13041 let registry = Arc::new(Registry::default());
13042 let forwarding = Arc::new(ForwardingTable::default());
13043 let handler =
13044 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13045 let (ctx, route_channel, route_epoch) =
13046 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
13047 let responses = handler
13048 .handle_route_poll(
13049 &ctx,
13050 route_poll_frame(42, PollKind::Liveness, route_channel),
13051 route_channel,
13052 route_epoch,
13053 PollKind::Liveness,
13054 )
13055 .unwrap();
13056
13057 assert_route_poll_liveness(&responses[0], true);
13058 }
13059
13060 #[test]
13061 fn liveness_poll_untracked_process_source_uses_bound_route() {
13062 let registry = Arc::new(Registry::default());
13063 let forwarding = Arc::new(ForwardingTable::default());
13064 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
13065 let handler =
13066 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13067 .with_process_liveness(process_liveness);
13068 let (ctx, route_channel, route_epoch) =
13069 bind_liveness_route(®istry, &forwarding, "aft-untracked");
13070 let responses = handler
13071 .handle_route_poll(
13072 &ctx,
13073 route_poll_frame(43, PollKind::Liveness, route_channel),
13074 route_channel,
13075 route_epoch,
13076 PollKind::Liveness,
13077 )
13078 .unwrap();
13079
13080 assert_route_poll_liveness(&responses[0], true);
13081 }
13082
13083 #[tokio::test]
13084 async fn unknown_op_returns_unknown_control_op() {
13085 let handler = ControlHandler::default();
13086 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
13087 let request = Frame::build(
13088 FrameType::Request,
13089 control_flags(),
13090 0,
13091 0,
13092 55,
13093 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
13094 )
13095 .unwrap();
13096
13097 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13098
13099 assert_eq!(response.len(), 1);
13100 assert_eq!(response[0].header.ty, FrameType::Error);
13101 assert_eq!(response[0].header.corr, 55);
13102 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
13103 }
13104
13105 #[tokio::test]
13106 async fn supervisor_provenance_rejects_unknown_exact_module() {
13107 let handler = ControlHandler::default();
13108 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
13109 let request = Frame::build(
13110 FrameType::Request,
13111 control_flags(),
13112 0,
13113 0,
13114 57,
13115 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
13116 )
13117 .unwrap();
13118
13119 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13120
13121 assert_eq!(response.len(), 1);
13122 assert_eq!(response[0].header.ty, FrameType::Error);
13123 assert_eq!(response[0].header.corr, 57);
13124 let error = parse_error(&response[0]);
13125 assert_eq!(error["code"], "unknown_module");
13126 assert_eq!(error["message"], "module_id 'missing' is not supervised");
13127 }
13128
13129 #[test]
13130 fn provenance_probe_override_keeps_handler_tests_deterministic() {
13131 let expected = subc_control::RunningImageAgreement::Unavailable {
13132 reason: subc_control::RunningImageUnavailableReason::HashFailed,
13133 };
13134 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
13135 assert_eq!(handler.provenance_probe_override, Some(expected));
13136 }
13137
13138 #[test]
13139 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
13140 let verdict = reload_verdict(
13141 std::path::Path::new("/bin/new"),
13142 Some(std::path::Path::new("/bin/old")),
13143 subc_control::RunningImageAgreement::Unavailable {
13144 reason: subc_control::RunningImageUnavailableReason::HashFailed,
13145 },
13146 );
13147 assert!(matches!(
13148 verdict.path,
13149 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
13150 if configured == std::path::Path::new("/bin/new")
13151 && spawned_from == std::path::Path::new("/bin/old")
13152 ));
13153 }
13154
13155 #[test]
13156 fn launch_spec_without_running_digest_is_unavailable_not_match() {
13157 use subc_control::{
13158 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
13159 };
13160 let configured = crate::supervise::ModuleSpec {
13161 module_id: "legacy".into(),
13162 program: "/module".into(),
13163 args: Vec::new(),
13164 env: Vec::new(),
13165 reserved: false,
13166 reserved_prefixes: Vec::new(),
13167 protocol: subc_control::ModuleProtocol::Subc,
13168 overlap: Default::default(),
13169 };
13170 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
13171 assert_eq!(
13172 launch_spec_agreement(&configured, false, Some(&digest)),
13173 Agreement::Unavailable {
13174 reason: Reason::NotRunning
13175 }
13176 );
13177 assert_eq!(
13178 launch_spec_agreement(&configured, true, None),
13179 Agreement::Unavailable {
13180 reason: Reason::SpawnedSpecUnavailable
13181 }
13182 );
13183 assert_eq!(
13184 launch_spec_agreement(&configured, true, Some(&digest)),
13185 Agreement::Match
13186 );
13187 }
13188
13189 #[cfg(windows)]
13190 #[test]
13191 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
13192 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
13193 let configured = dir.join("module.exe");
13194 std::fs::write(&configured, b"executable path fixture").unwrap();
13195 let spawned = std::fs::canonicalize(&configured).unwrap();
13196 assert_ne!(
13197 configured, spawned,
13198 "fixture must exercise different path spellings"
13199 );
13200 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
13201 let lower_case = configured.to_string_lossy().to_lowercase();
13202 for spelling in [
13203 &configured,
13204 Path::new(&forward_slashes),
13205 Path::new(&lower_case),
13206 ] {
13207 let verdict = reload_verdict(
13208 spelling,
13209 Some(&spawned),
13210 subc_control::RunningImageAgreement::Unavailable {
13211 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13212 },
13213 );
13214 assert_eq!(
13215 verdict.path,
13216 ReloadPathAgreement::Match,
13217 "configured {spelling:?}, spawned {spawned:?}"
13218 );
13219 }
13220 }
13221
13222 #[cfg(windows)]
13223 #[test]
13224 fn windows_reload_path_keeps_different_existing_program_pending() {
13225 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
13226 let old = dir.join("old.exe");
13227 let new = dir.join("new.exe");
13228 std::fs::write(&old, b"same executable content").unwrap();
13229 std::fs::copy(&old, &new).unwrap();
13230 let spawned = std::fs::canonicalize(&old).unwrap();
13231 let image = subc_control::RunningImageAgreement::Unavailable {
13232 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13233 };
13234 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
13235 assert!(
13236 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
13237 if configured == new && spawned_from == spawned)
13238 );
13239 let missing = dir.join("missing.exe");
13240 assert!(matches!(
13241 reload_verdict(&missing, Some(&spawned), image).path,
13242 ReloadPathAgreement::Mismatch { .. }
13243 ));
13244 }
13245
13246 #[test]
13247 fn reload_verdict_detects_replaced_image_at_same_path() {
13248 let image = subc_control::RunningImageAgreement::Mismatch {
13249 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
13250 digest: "old".into(),
13251 },
13252 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
13253 digest: "new".into(),
13254 },
13255 };
13256 let verdict = reload_verdict(
13257 std::path::Path::new("/bin/same"),
13258 Some(std::path::Path::new("/bin/same")),
13259 image.clone(),
13260 );
13261 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13262 assert_eq!(verdict.image, image);
13263 }
13264
13265 #[test]
13266 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
13267 let image = subc_control::RunningImageAgreement::Unavailable {
13268 reason: subc_control::RunningImageUnavailableReason::NotRunning,
13269 };
13270 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
13271 assert_eq!(
13272 verdict.path,
13273 subc_control::ReloadPathAgreement::Unavailable {
13274 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
13275 }
13276 );
13277 assert_eq!(verdict.image, image);
13278
13279 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
13280 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13281 };
13282 let verdict = reload_verdict(
13283 std::path::Path::new("/bin/same"),
13284 Some(std::path::Path::new("/bin/same")),
13285 unconfirmed.clone(),
13286 );
13287 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13288 assert_eq!(verdict.image, unconfirmed);
13289 }
13290
13291 #[test]
13292 fn reload_verdict_preserves_each_image_unavailability_reason() {
13293 use subc_control::RunningImageUnavailableReason as Reason;
13294
13295 for reason in [
13296 Reason::NotRunning,
13297 Reason::UnsupportedPlatform,
13298 Reason::RunningExecutableUnreadable,
13299 Reason::SpawnedPathUnreadable,
13300 Reason::HashFailed,
13301 Reason::ProcessIdentityUnconfirmed,
13302 Reason::Unknown("future_probe_reason".to_string()),
13303 ] {
13304 let image = subc_control::RunningImageAgreement::Unavailable {
13305 reason: reason.clone(),
13306 };
13307 let verdict = reload_verdict(
13308 std::path::Path::new("/bin/same"),
13309 Some(std::path::Path::new("/bin/same")),
13310 image.clone(),
13311 );
13312 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13313 assert_eq!(verdict.image, image, "{reason:?}");
13314 }
13315 }
13316
13317 #[tokio::test]
13318 async fn malformed_control_bodies_return_invalid_control_body() {
13319 let handler = ControlHandler::default();
13320 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
13321
13322 for (corr, body) in [
13323 (56, br#"{"route_channel":1}"#.as_slice()),
13324 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
13325 (
13326 58,
13327 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
13328 ),
13329 ] {
13330 let request = Frame::build(
13331 FrameType::Request,
13332 control_flags(),
13333 0,
13334 0,
13335 corr,
13336 body.to_vec(),
13337 )
13338 .unwrap();
13339 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13340
13341 assert_eq!(response.len(), 1);
13342 assert_eq!(response[0].header.ty, FrameType::Error);
13343 assert_eq!(response[0].header.corr, corr);
13344 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
13345 }
13346 }
13347
13348 #[tokio::test]
13349 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
13350 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13351 let registry = Arc::new(Registry::default());
13352 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13353 let router = Router::with_control_handler(Arc::clone(&control));
13354 let connection = router.begin_connection();
13355 let (ctx, mut rx) = route_ctx(connection.id());
13356
13357 router
13358 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
13359 .await
13360 .unwrap();
13361 let response = rx.recv().await.unwrap();
13362 let ack = parse_ack(&response);
13363 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13364 let channel = 1;
13365
13366 let goodbye =
13367 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
13368 router.route_for_connection(&ctx, goodbye).await.unwrap();
13369 assert!(rx.try_recv().is_err());
13370 assert!(registry.get_module("aft").unwrap().is_none());
13371
13372 router
13373 .route_for_connection(&ctx, channel_request(channel, 13))
13374 .await
13375 .unwrap();
13376 let error_frame = rx.recv().await.unwrap();
13377 assert_eq!(error_frame.header.ty, FrameType::Error);
13378 assert_eq!(error_frame.header.channel, channel);
13379 let captured = crate::router::test_log::captured_logs(&logs);
13380 assert_eq!(
13381 captured
13382 .lines()
13383 .filter(|line| {
13384 line.contains("module_id=aft")
13385 && line.contains("reason=explicit_goodbye")
13386 && line.contains("module registration ended")
13387 })
13388 .count(),
13389 1,
13390 "unexpected GOODBYE registry log: {captured}"
13391 );
13392 }
13393
13394 #[tokio::test]
13395 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
13396 let registry = Arc::new(Registry::default());
13397 let handler = ControlHandler::new(registry).with_capability_config(
13398 [("prov".to_string(), true), ("cons".to_string(), true)],
13399 BTreeMap::new(),
13400 );
13401 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
13402 register_capability_manifest(
13403 &handler,
13404 &provider_ctx,
13405 &mut provider_rx,
13406 capability_manifest("prov", &["thing/v1"], &[]),
13407 1,
13408 )
13409 .await;
13410 let mut consumer = capability_manifest("cons", &[], &[]);
13411 consumer.capabilities.as_mut().unwrap().requires.push(
13412 subc_protocol::manifest::CapabilityRequirement {
13413 capability: "thing/v1".to_string(),
13414 need: subc_protocol::manifest::CapabilityNeed::Required,
13415 },
13416 );
13417 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
13418 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
13419 assert_eq!(
13420 handler.capability_evaluator.verdict("cons", "thing/v1"),
13421 Some(CapabilityVerdict::Provided)
13422 );
13423 let (mut client_rx, _) = open_route_for_capability_test(
13424 &handler,
13425 &provider_ctx,
13426 &mut provider_rx,
13427 703,
13428 3,
13429 "prov",
13430 None,
13431 )
13432 .await;
13433 let goodbye =
13434 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13435 handler
13436 .handle_control_frame(&provider_ctx, goodbye)
13437 .await
13438 .unwrap();
13439 assert_eq!(
13440 handler.capability_evaluator.verdict("cons", "thing/v1"),
13441 Some(CapabilityVerdict::NeverProvided)
13442 );
13443 let closed = client_rx
13444 .try_recv()
13445 .expect("GOODBYE pushes route.closed before route GOODBYE");
13446 assert!(
13447 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
13448 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
13449 );
13450 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
13451 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
13452 }
13453
13454 struct RouteClosedFixture {
13455 handler: ControlHandler,
13456 #[cfg(unix)]
13457 supervisor: Supervisor,
13458 module_ctx: RouteCtx,
13459 #[cfg(unix)]
13460 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13461 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13462 route: Arc<crate::forwarding::RouteBinding>,
13463 }
13464
13465 async fn route_closed_fixture() -> RouteClosedFixture {
13466 let registry = Arc::new(Registry::default());
13467 let handler = ControlHandler::new(Arc::clone(®istry));
13468 #[cfg(unix)]
13469 let supervisor = Supervisor::new(registry, RestartPolicy::default())
13470 .with_forwarding(Arc::clone(&handler.forwarding));
13471 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
13472 register_capability_manifest(
13473 &handler,
13474 &module_ctx,
13475 &mut module_rx,
13476 capability_manifest("prov", &[], &[]),
13477 1,
13478 )
13479 .await;
13480 let (client_rx, bind) = open_route_for_capability_test(
13481 &handler,
13482 &module_ctx,
13483 &mut module_rx,
13484 702,
13485 2,
13486 "prov",
13487 None,
13488 )
13489 .await;
13490 let ModuleControlRequest::RouteBind {
13491 route_channel: channel,
13492 epoch: route_epoch,
13493 ..
13494 } = bind
13495 else {
13496 panic!("expected route.bind");
13497 };
13498 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
13499 handler
13500 .forwarding
13501 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
13502 .unwrap()
13503 else {
13504 panic!("the acknowledged route must be bound");
13505 };
13506 RouteClosedFixture {
13507 handler,
13508 #[cfg(unix)]
13509 supervisor,
13510 module_ctx,
13511 #[cfg(unix)]
13512 module_rx,
13513 client_rx,
13514 route,
13515 }
13516 }
13517
13518 impl RouteClosedFixture {
13519 #[cfg(unix)]
13520 async fn cut(&mut self) {
13521 let notices = async {
13522 let mut draining = self.module_rx.recv().await.unwrap();
13523 assert!(matches!(
13524 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
13525 &draining.body
13526 )
13527 .unwrap(),
13528 subc_protocol::session::ModuleControlCommand::Draining {
13529 reason: RouteCloseReason::Restart,
13530 ..
13531 }
13532 ));
13533 draining.flushed.take().unwrap().send(()).unwrap();
13534 let mut closing = self.client_rx.recv().await.unwrap();
13535 assert!(matches!(
13536 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
13537 ClientControlPush::RouteClosing {
13538 reason: RouteCloseReason::Restart,
13539 ..
13540 }
13541 ));
13542 closing.flushed.take().unwrap().send(()).unwrap();
13543 };
13544 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
13545 result.unwrap();
13546 }
13547
13548 fn disconnect(&mut self) -> ClientControlPush {
13549 self.handler
13550 .cleanup_connection(self.module_ctx.connection_id)
13551 .unwrap();
13552 let closed = self.client_rx.try_recv().unwrap();
13553 assert_eq!(closed.header.ty, FrameType::Push);
13554 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
13555 assert!(matches!(
13556 &push,
13557 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
13558 if channels == &[self.route.client_channel]
13559 ));
13560 assert_eq!(
13561 self.client_rx.try_recv().unwrap().header.ty,
13562 FrameType::Goodbye
13563 );
13564 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
13565 push
13566 }
13567 }
13568
13569 #[cfg(unix)]
13570 #[tokio::test(start_paused = true)]
13571 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
13572 let mut fixture = route_closed_fixture().await;
13573 fixture.cut().await;
13574 assert!(matches!(
13575 fixture.disconnect(),
13576 ClientControlPush::RouteClosed {
13577 reason: RouteCloseReason::Restart,
13578 drained: true,
13579 ..
13580 }
13581 ));
13582 }
13583
13584 #[cfg(unix)]
13585 #[tokio::test(start_paused = true)]
13586 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
13587 let mut fixture = route_closed_fixture().await;
13588 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
13589 let started = tokio::time::Instant::now();
13590 fixture.cut().await;
13591 assert!(started.elapsed() >= Duration::from_secs(2));
13592 assert_eq!(fixture.route.flow.in_flight(), 1);
13593 assert!(matches!(
13594 fixture.disconnect(),
13595 ClientControlPush::RouteClosed {
13596 reason: RouteCloseReason::Restart,
13597 drained: false,
13598 ..
13599 }
13600 ));
13601 }
13602
13603 #[tokio::test]
13604 async fn module_connection_crash_reports_crash_and_not_drained() {
13605 let mut fixture = route_closed_fixture().await;
13606 assert!(matches!(
13610 fixture.disconnect(),
13611 ClientControlPush::RouteClosed {
13612 reason: RouteCloseReason::Crash,
13613 drained: false,
13614 ..
13615 }
13616 ));
13617 }
13618
13619 #[tokio::test]
13620 async fn dropping_router_connection_releases_registration() {
13621 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13622 let registry = Arc::new(Registry::default());
13623 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13624 let router = Router::with_control_handler(Arc::clone(&control));
13625 let connection = router.begin_connection();
13626 let connection_id = connection.id();
13627 let (ctx, mut rx) = route_ctx(connection_id);
13628
13629 router
13630 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
13631 .await
13632 .unwrap();
13633 let response = rx.recv().await.unwrap();
13634 let ack = parse_ack(&response);
13635 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13636 assert!(registry.get_module("aft").unwrap().is_some());
13637
13638 drop(connection);
13639
13640 assert!(registry.get_module("aft").unwrap().is_none());
13641 assert_eq!(registry.active_registration_count().unwrap(), 0);
13642
13643 control
13644 .cleanup_connection(ConnectionId::new(u64::MAX))
13645 .unwrap();
13646 let captured = crate::router::test_log::captured_logs(&logs);
13647 println!("captured registry lifecycle logs:\n{captured}");
13648 let events: Vec<_> = captured
13649 .lines()
13650 .filter(|line| {
13651 line.contains("module registered module_id=aft ")
13652 || line.contains("module registration ended")
13653 })
13654 .collect();
13655 assert_eq!(
13656 events.len(),
13657 2,
13658 "unexpected registry lifecycle logs: {captured}"
13659 );
13660 assert!(events[0].contains("module registered module_id=aft "));
13661 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
13662 assert!(events[1].contains(&format!(
13663 "module_id=aft connection_id={}",
13664 connection_id.get()
13665 )));
13666 assert!(events[1].contains("reason=connection_closed"));
13667 assert!(events[1].contains("module registration ended"));
13668 assert_eq!(
13669 captured
13670 .lines()
13671 .filter(|line| line.contains("module registered module_id=aft "))
13672 .count(),
13673 1,
13674 "legacy registration admission line must appear once: {captured}"
13675 );
13676 }
13677
13678 #[tokio::test]
13679 async fn hello_registration_keeps_legacy_module_registered_line_once() {
13680 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13681 let registry = Arc::new(Registry::default());
13682 let control = ControlHandler::new(Arc::clone(®istry));
13683 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
13684 hello_via_sink(
13685 &control,
13686 &ctx,
13687 &mut rx,
13688 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
13689 )
13690 .await;
13691
13692 let captured = crate::router::test_log::captured_logs(&logs);
13693 let admissions: Vec<_> = captured
13694 .lines()
13695 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
13696 .collect();
13697 assert_eq!(
13698 admissions.len(),
13699 1,
13700 "legacy admission line must remain exactly once: {captured}"
13701 );
13702 assert!(admissions[0].contains("routable_provider=true"));
13703 assert!(admissions[0].contains("connection_id=777"));
13704 }
13705
13706 fn capability_manifest(
13707 module_id: &str,
13708 provides: &[&str],
13709 must_never_reach: &[&str],
13710 ) -> ModuleManifest {
13711 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
13712 manifest.capabilities = Some(CapabilityDeclarations {
13713 provides: provides
13714 .iter()
13715 .map(|capability| (*capability).to_string())
13716 .collect(),
13717 requires: Vec::new(),
13718 must_never_reach: must_never_reach
13719 .iter()
13720 .map(|capability| (*capability).to_string())
13721 .collect(),
13722 });
13723 manifest
13724 }
13725
13726 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
13727 Frame::build(
13728 FrameType::Hello,
13729 control_flags(),
13730 0,
13731 0,
13732 corr,
13733 serde_json::to_vec(&ModuleHelloBody {
13734 protocol_ver: manifest.protocol_ver,
13735 manifest,
13736 control_ops: None,
13737 launch_nonce: None,
13738 })
13739 .expect("capability test HELLO serializes"),
13740 )
13741 .expect("capability test HELLO frame builds")
13742 }
13743
13744 fn catalog_update_with_capabilities_frame(
13745 corr: u64,
13746 capabilities: CapabilityDeclarations,
13747 ) -> Frame {
13748 Frame::build(
13749 FrameType::Request,
13750 control_flags(),
13751 0,
13752 0,
13753 corr,
13754 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13755 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
13756 capabilities: Some(capabilities),
13757 ready: None,
13758 })
13759 .expect("capability catalog.update serializes"),
13760 )
13761 .expect("capability catalog.update frame builds")
13762 }
13763
13764 async fn register_capability_manifest(
13765 handler: &ControlHandler,
13766 ctx: &RouteCtx,
13767 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13768 manifest: ModuleManifest,
13769 corr: u64,
13770 ) {
13771 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
13772 }
13773
13774 async fn open_route_for_capability_test(
13775 handler: &ControlHandler,
13776 target_ctx: &RouteCtx,
13777 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13778 client_connection_id: u64,
13779 corr: u64,
13780 target_module_id: &str,
13781 consumer_identity: Option<ConsumerIdentity>,
13782 ) -> (
13783 mpsc::Receiver<crate::router::OutboundFrame>,
13784 ModuleControlRequest,
13785 ) {
13786 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
13787 let route_handler = handler.clone();
13788 let target_module_id = target_module_id.to_string();
13789 let route_task = tokio::spawn(async move {
13790 route_handler
13791 .handle_control_frame(
13792 &client_ctx,
13793 route_open_frame_with_admission_facts(
13794 corr,
13795 &target_module_id,
13796 unique_project_root("admission-facts"),
13797 consumer_identity,
13798 None,
13799 ),
13800 )
13801 .await
13802 .expect("capability test route.open succeeds")
13803 });
13804 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
13805 .await
13806 .expect("capability test route.open must reach route.bind")
13807 .expect("target control receiver stays open");
13808 let bind_request: ModuleControlRequest =
13809 serde_json::from_slice(&bind.body).expect("route.bind decodes");
13810 handler
13811 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
13812 .await
13813 .expect("capability test route.bind ACK succeeds");
13814 assert!(route_task.await.expect("route.open task joins").is_empty());
13815 let opened = client_rx
13816 .recv()
13817 .await
13818 .expect("successful route.open publishes a response");
13819 assert!(matches!(
13820 serde_json::from_slice::<ClientControlResponse>(&opened.body),
13821 Ok(ClientControlResponse::RouteOpen { .. })
13822 ));
13823 (client_rx, bind_request)
13824 }
13825
13826 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
13827 assert_eq!(frame.header.ty, FrameType::Push);
13828 assert_eq!(frame.header.channel, 0);
13829 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
13830 .expect("route.closed control push decodes");
13831 let ClientControlPush::RouteClosed { channels, .. } = &push else {
13832 panic!("expected route.closed");
13833 };
13834 assert_eq!(channels.len(), 1, "exactly one violating route closed");
13835 let channels = channels.clone();
13836 assert_eq!(
13837 push,
13838 ClientControlPush::RouteClosed {
13839 module_id: target_module_id.to_string(),
13840 channels,
13841 reason: RouteCloseReason::CapabilityDenied,
13842 drained: false,
13843 abandoned: 0,
13844 excluded_subscriptions: 0,
13845 terminal: Some(false),
13846 }
13847 );
13848 }
13849
13850 #[tokio::test]
13851 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
13852 let registry = Arc::new(Registry::default());
13853 let forwarding = Arc::new(ForwardingTable::default());
13854 let supervisor = SupervisorHandle::new();
13855 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13856 let handler =
13857 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13858 .with_supervisor(supervisor);
13859 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
13860 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
13861 register_capability_manifest(
13862 &handler,
13863 &target_ctx,
13864 &mut target_rx,
13865 capability_manifest("target", &["credentials-provider/v1"], &[]),
13866 1,
13867 )
13868 .await;
13869 register_capability_manifest(
13870 &handler,
13871 &opener_ctx,
13872 &mut opener_rx,
13873 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13874 2,
13875 )
13876 .await;
13877
13878 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
13879 let replies = handler
13880 .handle_control_frame(
13881 &client_ctx,
13882 route_open_frame_with_admission_facts(
13883 3,
13884 "target",
13885 unique_project_root("admission-facts"),
13886 Some(ConsumerIdentity {
13887 module_id: "opener".to_string(),
13888 launch_nonce: "opener-nonce".to_string(),
13889 }),
13890 None,
13891 ),
13892 )
13893 .await
13894 .expect("denied route.open returns a typed frame");
13895 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13896 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13897 assert!(
13898 target_rx.try_recv().is_err(),
13899 "forbidden route.open must not relay route.bind"
13900 );
13901 }
13902
13903 #[tokio::test]
13904 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
13905 let registry = Arc::new(Registry::default());
13906 let forwarding = Arc::new(ForwardingTable::default());
13907 let supervisor = SupervisorHandle::new();
13908 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13909 let handler =
13910 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13911 .with_supervisor(supervisor);
13912 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
13913 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
13914 register_capability_manifest(
13915 &handler,
13916 &target_ctx,
13917 &mut target_rx,
13918 capability_manifest("target", &["credentials-provider/v1"], &[]),
13919 1,
13920 )
13921 .await;
13922 register_capability_manifest(
13923 &handler,
13924 &old_opener_ctx,
13925 &mut old_opener_rx,
13926 capability_manifest("opener", &[], &[]),
13927 2,
13928 )
13929 .await;
13930 let (mut client_rx, _) = open_route_for_capability_test(
13931 &handler,
13932 &target_ctx,
13933 &mut target_rx,
13934 712,
13935 3,
13936 "target",
13937 Some(ConsumerIdentity {
13938 module_id: "opener".to_string(),
13939 launch_nonce: "opener-nonce".to_string(),
13940 }),
13941 )
13942 .await;
13943 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13944
13945 handler
13946 .cleanup_connection(old_opener_ctx.connection_id)
13947 .expect("old opener registration cleans up");
13948 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
13949 register_capability_manifest(
13950 &handler,
13951 &new_opener_ctx,
13952 &mut new_opener_rx,
13953 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13954 4,
13955 )
13956 .await;
13957
13958 assert_capability_denied_push(
13959 client_rx
13960 .try_recv()
13961 .expect("HELLO deny addition must emit route.closed")
13962 .frame,
13963 "target",
13964 );
13965 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13966 assert!(matches!(
13967 target_rx.try_recv(),
13968 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13969 ));
13970 }
13971
13972 #[tokio::test]
13973 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
13974 let registry = Arc::new(Registry::default());
13975 let forwarding = Arc::new(ForwardingTable::default());
13976 let supervisor = SupervisorHandle::new();
13977 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13978 let handler =
13979 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13980 .with_supervisor(supervisor);
13981 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
13982 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
13983 register_capability_manifest(
13984 &handler,
13985 &target_ctx,
13986 &mut target_rx,
13987 capability_manifest("target", &[], &[]),
13988 1,
13989 )
13990 .await;
13991 register_capability_manifest(
13992 &handler,
13993 &opener_ctx,
13994 &mut opener_rx,
13995 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13996 2,
13997 )
13998 .await;
13999 let (mut client_rx, _) = open_route_for_capability_test(
14000 &handler,
14001 &target_ctx,
14002 &mut target_rx,
14003 722,
14004 3,
14005 "target",
14006 Some(ConsumerIdentity {
14007 module_id: "opener".to_string(),
14008 launch_nonce: "opener-nonce".to_string(),
14009 }),
14010 )
14011 .await;
14012 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14013
14014 let replies = handler
14015 .handle_control_frame(
14016 &target_ctx,
14017 catalog_update_with_capabilities_frame(
14018 4,
14019 CapabilityDeclarations {
14020 provides: vec!["credentials-provider/v1".to_string()],
14021 requires: Vec::new(),
14022 must_never_reach: Vec::new(),
14023 },
14024 ),
14025 )
14026 .await
14027 .expect("claim catalog.update succeeds");
14028 assert!(matches!(
14029 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
14030 Ok(ModuleControlResponseToModule::CatalogUpdate {})
14031 ));
14032 assert_capability_denied_push(
14033 client_rx
14034 .try_recv()
14035 .expect("claim addition must emit route.closed")
14036 .frame,
14037 "target",
14038 );
14039 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14040 assert!(matches!(
14041 target_rx.try_recv(),
14042 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14043 ));
14044 }
14045
14046 #[tokio::test]
14047 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
14048 let registry = Arc::new(Registry::default());
14049 let forwarding = Arc::new(ForwardingTable::default());
14050 let supervisor = SupervisorHandle::new();
14051 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14052 let handler =
14053 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14054 .with_supervisor(supervisor);
14055 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
14056 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
14057 register_capability_manifest(
14058 &handler,
14059 &target_ctx,
14060 &mut target_rx,
14061 capability_manifest("target", &["credentials-provider/v1"], &[]),
14062 1,
14063 )
14064 .await;
14065 register_capability_manifest(
14066 &handler,
14067 &opener_ctx,
14068 &mut opener_rx,
14069 capability_manifest("opener", &[], &[]),
14070 2,
14071 )
14072 .await;
14073 let (mut client_rx, _) = open_route_for_capability_test(
14074 &handler,
14075 &target_ctx,
14076 &mut target_rx,
14077 732,
14078 3,
14079 "target",
14080 Some(ConsumerIdentity {
14081 module_id: "opener".to_string(),
14082 launch_nonce: "opener-nonce".to_string(),
14083 }),
14084 )
14085 .await;
14086 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14087
14088 handler
14089 .handle_control_frame(
14090 &target_ctx,
14091 catalog_update_with_capabilities_frame(
14092 4,
14093 CapabilityDeclarations {
14094 provides: Vec::new(),
14095 requires: Vec::new(),
14096 must_never_reach: Vec::new(),
14097 },
14098 ),
14099 )
14100 .await
14101 .expect("claim removal catalog.update succeeds");
14102 assert_eq!(
14103 forwarding.active_binding_count().unwrap(),
14104 1,
14105 "removing an attested target claim must leave the route census unchanged"
14106 );
14107 assert!(
14108 client_rx.try_recv().is_err(),
14109 "claim removal must not emit route.closed capability_denied"
14110 );
14111 assert!(
14112 target_rx.try_recv().is_err(),
14113 "claim removal must not send the target a route GOODBYE"
14114 );
14115 }
14116
14117 #[tokio::test]
14120 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
14121 let registry = Arc::new(Registry::default());
14122 let forwarding = Arc::new(ForwardingTable::default());
14123 let supervisor = SupervisorHandle::new();
14124 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14125 let handler =
14126 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14127 .with_supervisor(supervisor);
14128 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
14129 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
14130 register_capability_manifest(
14131 &handler,
14132 &target_ctx,
14133 &mut target_rx,
14134 capability_manifest("target", &["credentials-provider/v1"], &[]),
14135 1,
14136 )
14137 .await;
14138 register_capability_manifest(
14139 &handler,
14140 &opener_ctx,
14141 &mut opener_rx,
14142 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14143 2,
14144 )
14145 .await;
14146
14147 let (_client_rx, bind) = open_route_for_capability_test(
14148 &handler,
14149 &target_ctx,
14150 &mut target_rx,
14151 742,
14152 3,
14153 "target",
14154 None,
14155 )
14156 .await;
14157 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14158 panic!("direct scope-honesty route must bind");
14159 };
14160 assert_eq!(principal, Some(Principal::Direct));
14161 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14162 }
14163
14164 #[tokio::test]
14167 async fn must_never_reach_self_route_is_capability_forbidden() {
14168 let registry = Arc::new(Registry::default());
14169 let forwarding = Arc::new(ForwardingTable::default());
14170 let supervisor = SupervisorHandle::new();
14171 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
14172 let handler =
14173 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14174 .with_supervisor(supervisor);
14175 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
14176 register_capability_manifest(
14177 &handler,
14178 &self_ctx,
14179 &mut self_rx,
14180 capability_manifest(
14181 "self-provider",
14182 &["credentials-provider/v1"],
14183 &["credentials-provider/v1"],
14184 ),
14185 1,
14186 )
14187 .await;
14188
14189 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
14190 let replies = handler
14191 .handle_control_frame(
14192 &client_ctx,
14193 route_open_frame_with_admission_facts(
14194 2,
14195 "self-provider",
14196 unique_project_root("admission-facts"),
14197 Some(ConsumerIdentity {
14198 module_id: "self-provider".to_string(),
14199 launch_nonce: "self-nonce".to_string(),
14200 }),
14201 None,
14202 ),
14203 )
14204 .await
14205 .expect("self-route refusal returns a typed frame");
14206 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14207 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14208 assert!(
14209 self_rx.try_recv().is_err(),
14210 "self denial must not relay route.bind"
14211 );
14212 }
14213
14214 #[test]
14215 fn unsupported_channel_zero_frame_returns_error() {
14216 let handler = ControlHandler::default();
14217 let request = Frame::build(
14218 FrameType::Request,
14219 control_flags(),
14220 0,
14221 0,
14222 21,
14223 b"opaque".to_vec(),
14224 )
14225 .unwrap();
14226
14227 let response = handler
14228 .handle_control(ConnectionId::new(1), request)
14229 .unwrap();
14230
14231 assert_eq!(response[0].header.ty, FrameType::Error);
14232 assert_eq!(
14233 parse_error(&response[0])["code"],
14234 "unsupported_control_frame"
14235 );
14236 }
14237
14238 mod swap {
14243 use super::*;
14244
14245 const INCUMBENT: ConnectionId = ConnectionId::new(30);
14246 const CANDIDATE: ConnectionId = ConnectionId::new(40);
14247
14248 struct Swap {
14249 registry: Arc<Registry>,
14250 forwarding: Arc<ForwardingTable>,
14251 handler: ControlHandler,
14252 incumbent_ctx: RouteCtx,
14253 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14254 candidate_ctx: RouteCtx,
14255 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14256 }
14257
14258 async fn swap_with_incumbent() -> Swap {
14259 let registry = Arc::new(Registry::default());
14260 let forwarding = Arc::new(ForwardingTable::default());
14261 let handler =
14262 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14263 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
14264 hello_via_sink(
14265 &handler,
14266 &incumbent_ctx,
14267 &mut incumbent_rx,
14268 hello_frame("aft", PROTOCOL_VERSION, 7),
14269 )
14270 .await;
14271 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
14272 Swap {
14273 registry,
14274 forwarding,
14275 handler,
14276 incumbent_ctx,
14277 incumbent_rx,
14278 candidate_ctx,
14279 candidate_rx,
14280 }
14281 }
14282
14283 fn register_candidate(swap: &Swap, ready: Option<bool>) {
14284 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
14285 candidate_manifest.ready = ready;
14286 let registration = swap
14287 .registry
14288 .register_candidate_with_control_ops(
14289 candidate_manifest,
14290 PROTOCOL_VERSION,
14291 CANDIDATE,
14292 module_baseline_control_ops(),
14293 )
14294 .unwrap();
14295 swap.forwarding
14296 .register_candidate_module_connection(
14297 CANDIDATE,
14298 "aft".to_string(),
14299 PROTOCOL_VERSION,
14300 manifest_concurrency(®istration.manifest),
14301 swap.candidate_ctx.egress.clone(),
14302 )
14303 .unwrap();
14304 }
14305
14306 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
14307 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
14308 swap.registry.promote_candidate("aft").unwrap().unwrap();
14309 cutover.incumbent.unwrap()
14310 }
14311
14312 #[tokio::test]
14321 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
14322 let mut swap = swap_with_incumbent().await;
14323 let hold = Arc::new(std::sync::Barrier::new(2));
14324 let handler = swap
14325 .handler
14326 .clone()
14327 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14328 handler
14329 .supervisor
14330 .open_swap("aft", "candidate-nonce".to_string());
14331 let hello = {
14332 let handler = handler.clone();
14333 let candidate_ctx = swap.candidate_ctx.clone();
14334 std::thread::spawn(move || {
14335 handler
14336 .handle_hello(
14337 candidate_ctx.connection_id,
14338 Some(candidate_ctx.egress.clone()),
14339 hello_frame_with_nonce(
14340 "aft",
14341 PROTOCOL_VERSION,
14342 8,
14343 Some("candidate-nonce"),
14344 ),
14345 )
14346 .unwrap()
14347 })
14348 };
14349
14350 hold.wait();
14351 assert!(swap
14354 .registry
14355 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
14356 .unwrap()
14357 .is_some());
14358 assert!(swap
14359 .registry
14360 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14361 .unwrap()
14362 .is_none());
14363 assert!(swap
14364 .forwarding
14365 .module_endpoint_for_connection(CANDIDATE)
14366 .unwrap()
14367 .is_none());
14368
14369 hold.wait();
14370 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
14371 assert_eq!(
14372 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
14373 FrameType::HelloAck
14374 );
14375 let published = swap
14376 .registry
14377 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14378 .unwrap()
14379 .expect("candidate is published once its HELLO completes");
14380 assert_eq!(published.connection_id, CANDIDATE);
14381 assert!(swap
14382 .forwarding
14383 .module_endpoint_for_connection(CANDIDATE)
14384 .unwrap()
14385 .is_some());
14386 }
14387
14388 fn keyed_total(counters: &Value, key: &str) -> u64 {
14389 counters[key]
14390 .as_object()
14391 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
14392 .unwrap_or(0)
14393 }
14394
14395 #[tokio::test]
14396 async fn replacement_logs_old_end_and_new_admission_once() {
14397 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14398 let mut swap = swap_with_incumbent().await;
14399 swap.handler
14400 .supervisor
14401 .open_swap("aft", "candidate-nonce".to_string());
14402 hello_via_sink(
14403 &swap.handler,
14404 &swap.candidate_ctx,
14405 &mut swap.candidate_rx,
14406 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
14407 )
14408 .await;
14409 cutover(&swap);
14410 swap.handler.cleanup_connection(INCUMBENT).unwrap();
14411
14412 let captured = crate::router::test_log::captured_logs(&logs);
14413 println!("captured replacement registry logs:\n{captured}");
14414 let new_admissions: Vec<_> = captured
14415 .lines()
14416 .filter(|line| {
14417 line.contains("connection_id=40")
14418 && line.contains("swap candidate registered; not routable until cutover")
14419 })
14420 .collect();
14421 assert_eq!(
14422 new_admissions.len(),
14423 1,
14424 "unexpected admission logs: {captured}"
14425 );
14426 assert!(new_admissions[0].contains("module_id=aft"));
14427 assert!(new_admissions[0].contains("connection_id=40"));
14428 assert!(
14429 new_admissions[0].contains("swap candidate registered; not routable until cutover")
14430 );
14431 assert!(new_admissions[0].contains("ready=true"));
14432
14433 let old_admissions: Vec<_> = captured
14434 .lines()
14435 .filter(|line| {
14436 line.contains("module registered module_id=aft ")
14437 && line.contains("connection_id=30")
14438 })
14439 .collect();
14440 assert_eq!(
14441 old_admissions.len(),
14442 1,
14443 "unexpected incumbent admission: {captured}"
14444 );
14445
14446 let old_ends: Vec<_> = captured
14447 .lines()
14448 .filter(|line| {
14449 line.contains("connection_id=30") && line.contains("module registration ended")
14450 })
14451 .collect();
14452 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
14453 assert!(old_ends[0].contains("module_id=aft"));
14454 assert!(old_ends[0].contains("connection_id=30"));
14455 assert!(old_ends[0].contains("reason=replaced"));
14456 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
14457 assert_eq!(
14458 captured
14459 .lines()
14460 .filter(|line| line.contains("module registration promoted"))
14461 .count(),
14462 1,
14463 "unexpected promotion logs: {captured}"
14464 );
14465 }
14466
14467 #[tokio::test]
14473 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
14474 let mut swap = swap_with_incumbent().await;
14475 let handler = swap.handler.clone();
14476
14477 let cotenant = ConnectionId::new(31);
14479 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
14480 let (cotenant_task, cotenant_bind) = relay_route_open(
14481 &handler,
14482 cotenant,
14483 &cotenant_ctx.egress,
14484 &mut swap.incumbent_rx,
14485 100,
14486 "aft",
14487 "swap-cotenant",
14488 )
14489 .await;
14490 handler
14491 .handle_control_frame(
14492 &swap.incumbent_ctx,
14493 route_bind_ack(cotenant_bind.header.corr),
14494 )
14495 .await
14496 .unwrap();
14497 assert!(cotenant_task.await.unwrap().is_empty());
14498 let (cotenant_channel, cotenant_epoch) =
14499 published_route(&cotenant_rx.recv().await.unwrap());
14500
14501 let caller = ConnectionId::new(32);
14503 let (caller_ctx, mut caller_rx) = route_ctx(caller);
14504 let (caller_task, caller_bind) = relay_route_open(
14505 &handler,
14506 caller,
14507 &caller_ctx.egress,
14508 &mut swap.incumbent_rx,
14509 101,
14510 "aft",
14511 "swap-caller",
14512 )
14513 .await;
14514 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
14515
14516 register_candidate(&swap, None);
14517 cutover(&swap);
14518
14519 let ack = handler
14521 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
14522 .await;
14523 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
14524 if module_loop_error.is_some() {
14525 handler.cleanup_connection(INCUMBENT).unwrap();
14528 }
14529
14530 assert!(
14532 cotenant_rx.try_recv().is_err(),
14533 "the co-tenant route on the incumbent was torn down by one late ack: \
14534 {module_loop_error:?}"
14535 );
14536 assert!(matches!(
14537 swap.forwarding
14538 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14539 .unwrap(),
14540 DataRoute::Client(DataRouteState::Bound(_))
14541 ));
14542 assert_eq!(module_loop_error, None);
14543 assert!(swap
14544 .registry
14545 .get_module_by_connection(INCUMBENT)
14546 .unwrap()
14547 .is_some());
14548
14549 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
14551 .await
14552 .expect("the incumbent is told to drop the abandoned binding")
14553 .unwrap()
14554 .frame;
14555 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14556 assert_eq!(goodbye.header.channel, abandoned_channel);
14557 assert_eq!(goodbye.header.epoch, abandoned_epoch);
14558 assert!(swap.incumbent_rx.try_recv().is_err());
14559
14560 let response = caller_task.await.unwrap();
14562 assert_eq!(response.len(), 1);
14563 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14564 assert!(caller_rx.try_recv().is_err());
14565
14566 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
14570 let counters = handler.counters().snapshot();
14571 assert_eq!(
14572 keyed_total(&counters, "route_open_accepted_by_principal"),
14573 1
14574 );
14575 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
14576 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
14577 }
14578
14579 #[tokio::test]
14583 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
14584 let mut swap = swap_with_incumbent().await;
14585 register_candidate(&swap, None);
14586 let incumbent = cutover(&swap);
14587 swap.forwarding
14588 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
14589 .unwrap()
14590 .expect("the incumbent is still registered");
14591
14592 let client = ConnectionId::new(33);
14593 let (client_ctx, mut client_rx) = route_ctx(client);
14594 let route_handler = swap.handler.clone();
14595 let open_ctx = RouteCtx {
14596 connection_id: client,
14597 egress: client_ctx.egress.clone(),
14598 };
14599 let mut route_task = tokio::spawn(async move {
14600 route_handler
14601 .handle_control_frame(
14602 &open_ctx,
14603 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
14604 )
14605 .await
14606 .unwrap()
14607 });
14608 let bind = tokio::select! {
14609 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
14610 response = &mut route_task => {
14611 let response = response.unwrap();
14612 panic!(
14613 "post-cutover route.open was refused instead of relayed to the candidate: {}",
14614 parse_error(&response[0])["code"]
14615 );
14616 }
14617 };
14618 swap.handler
14619 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
14620 .await
14621 .unwrap();
14622 assert!(route_task.await.unwrap().is_empty());
14623 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
14624 match swap
14625 .forwarding
14626 .lookup_data_route(client, channel, epoch)
14627 .unwrap()
14628 {
14629 DataRoute::Client(DataRouteState::Bound(route)) => {
14630 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
14631 }
14632 other => panic!("expected a bound route on the candidate, got {other:?}"),
14633 }
14634 assert!(swap.incumbent_rx.try_recv().is_err());
14635 }
14636
14637 #[tokio::test]
14642 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
14643 let swap = swap_with_incumbent().await;
14644 register_candidate(&swap, Some(false));
14645 let update = Frame::build(
14646 FrameType::Request,
14647 control_flags(),
14648 0,
14649 0,
14650 55,
14651 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14652 provides: manifest("aft", PROTOCOL_VERSION).provides,
14653 capabilities: None,
14654 ready: Some(true),
14655 })
14656 .unwrap(),
14657 )
14658 .unwrap();
14659
14660 let replies = swap
14661 .handler
14662 .handle_control_frame(&swap.candidate_ctx, update)
14663 .await
14664 .unwrap();
14665
14666 assert_eq!(replies.len(), 1);
14667 assert_eq!(
14668 replies[0].header.ty,
14669 FrameType::Response,
14670 "candidate catalog.update was refused: {:?}",
14671 serde_json::from_slice::<Value>(&replies[0].body).ok()
14672 );
14673 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
14674 assert_eq!(
14675 swap.registry
14676 .get_module("aft")
14677 .unwrap()
14678 .unwrap()
14679 .connection_id,
14680 INCUMBENT
14681 );
14682 }
14683 }
14684
14685 mod swap_admission {
14689 use super::*;
14690
14691 const INCUMBENT_NONCE: &str = "incumbent-nonce";
14692 const CANDIDATE_NONCE: &str = "candidate-nonce";
14693
14694 fn handler_with_incumbent(
14695 module_id: &str,
14696 reserved: bool,
14697 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
14698 let registry = Arc::new(Registry::default());
14699 let supervisor = SupervisorHandle::new();
14700 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
14701 if reserved {
14702 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
14703 }
14704 let handler =
14705 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
14706 let incumbent = handler
14707 .handle_control(
14708 ConnectionId::new(1),
14709 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
14710 )
14711 .unwrap();
14712 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
14713 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
14714 (registry, supervisor, handler)
14715 }
14716
14717 #[test]
14723 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
14724 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
14725
14726 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
14727 let replies = handler
14728 .handle_control(
14729 ConnectionId::new(connection),
14730 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
14731 )
14732 .unwrap();
14733 assert_eq!(replies[0].header.ty, FrameType::Error);
14734 assert_eq!(
14735 parse_error(&replies[0])["code"],
14736 "swap_token_invalid",
14737 "nonce {nonce:?}"
14738 );
14739 }
14740 assert!(registry.get_candidate("aft").unwrap().is_none());
14741 assert_eq!(
14742 registry.get_module("aft").unwrap().unwrap().connection_id,
14743 ConnectionId::new(1)
14744 );
14745
14746 let admitted = handler
14749 .handle_control(
14750 ConnectionId::new(4),
14751 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
14752 )
14753 .unwrap();
14754 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
14755 assert_eq!(
14756 registry
14757 .get_candidate("aft")
14758 .unwrap()
14759 .unwrap()
14760 .connection_id,
14761 ConnectionId::new(4)
14762 );
14763 assert_eq!(
14764 registry.get_module("aft").unwrap().unwrap().connection_id,
14765 ConnectionId::new(1),
14766 "the candidate must not take the active slot"
14767 );
14768 let replayed = handler
14769 .handle_control(
14770 ConnectionId::new(5),
14771 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
14772 )
14773 .unwrap();
14774 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
14775
14776 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
14780 let squatter = handler
14781 .handle_control(
14782 ConnectionId::new(6),
14783 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
14784 )
14785 .unwrap();
14786 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
14787 assert!(
14788 registry.get_module("aft").unwrap().is_none(),
14789 "a squatter took the active slot of an id being swapped"
14790 );
14791 }
14792
14793 #[test]
14798 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
14799 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
14800
14801 let replies = handler
14802 .handle_control(
14803 ConnectionId::new(2),
14804 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14805 )
14806 .unwrap();
14807
14808 assert_eq!(
14809 replies[0].header.ty,
14810 FrameType::HelloAck,
14811 "reserved candidate refused: {:?}",
14812 serde_json::from_slice::<Value>(&replies[0].body).ok()
14813 );
14814 assert_eq!(
14815 registry
14816 .get_candidate("vault")
14817 .unwrap()
14818 .unwrap()
14819 .connection_id,
14820 ConnectionId::new(2)
14821 );
14822 }
14823
14824 #[test]
14827 fn without_an_open_swap_the_ordinary_gates_decide() {
14828 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
14829 supervisor.close_swap("vault");
14830
14831 let candidate = handler
14832 .handle_control(
14833 ConnectionId::new(2),
14834 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14835 )
14836 .unwrap();
14837 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
14838 let duplicate = handler
14839 .handle_control(
14840 ConnectionId::new(3),
14841 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
14842 )
14843 .unwrap();
14844 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14845 assert!(registry.get_candidate("vault").unwrap().is_none());
14846 }
14847 }
14848
14849 mod scopes {
14853 mod idle_tests {
14854 use super::*;
14855 include!("scope_idle_tests.rs");
14856 }
14857 mod route_open_timings {
14858 use super::*;
14859 include!("route_open_timing_tests.rs");
14860 }
14861 mod activity {
14862 include!("activity_scope_tests.rs");
14863 }
14864 mod delta {
14865 include!("scope_handler_delta_tests.rs");
14866 }
14867 mod call_modes {
14868 include!("scope_call_mode_tests.rs");
14869 }
14870 #[tokio::test]
14871 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
14872 let rig = rig().await;
14873 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
14874 "s".into(),
14875 1,
14876 "tool".into(),
14877 ))
14878 .unwrap();
14879 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
14880 let (direct, _rx) = wide_ctx(999);
14881 for (ctx, expected) in [
14882 (&rig.owner, FrameType::Response),
14883 (&direct, FrameType::Error),
14884 ] {
14885 let frame =
14886 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
14887 .unwrap();
14888 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
14889 assert_eq!(replies[0].header.ty, expected);
14890 if expected == FrameType::Response {
14891 let reply: subc_control::ScopeServedByReply =
14892 serde_json::from_slice(&replies[0].body).unwrap();
14893 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
14894 } else {
14895 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
14896 }
14897 }
14898 let reply = call(
14899 &rig.handler,
14900 &rig.owner,
14901 &ModuleControlRequestFromModule::ScopeDescribe {
14902 owner: Principal::Reserved {
14903 module_id: OWNER.into(),
14904 },
14905 scope_ref: "s".into(),
14906 },
14907 )
14908 .await;
14909 assert!(matches!(
14910 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
14911 ModuleControlResponseToModule::ScopeDescribe { .. }
14912 ));
14913 }
14914
14915 use subc_protocol::scope::{
14916 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
14917 ScopeStatus,
14918 };
14919
14920 use super::*;
14921
14922 const OWNER: &str = "prefrontal-core";
14923
14924 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
14925 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
14926 }
14927
14928 async fn call(
14929 handler: &ControlHandler,
14930 ctx: &RouteCtx,
14931 request: &ModuleControlRequestFromModule,
14932 ) -> Frame {
14933 let body = serde_json::to_vec(request).unwrap();
14934 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
14935 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
14936 assert_eq!(replies.len(), 1, "{replies:?}");
14937 replies.pop().unwrap()
14938 }
14939
14940 async fn sync(
14941 handler: &ControlHandler,
14942 ctx: &RouteCtx,
14943 generation: u64,
14944 scopes: Vec<ScopeRecord>,
14945 ) -> Result<ModuleControlResponseToModule, String> {
14946 let reply = call(
14947 handler,
14948 ctx,
14949 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
14950 )
14951 .await;
14952 match reply.header.ty {
14953 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
14954 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
14955 }
14956 }
14957
14958 async fn describe(
14959 handler: &ControlHandler,
14960 ctx: &RouteCtx,
14961 owner: &str,
14962 scope_ref: &str,
14963 ) -> ModuleControlResponseToModule {
14964 let reply = call(
14965 handler,
14966 ctx,
14967 &ModuleControlRequestFromModule::ScopeDescribe {
14968 owner: Principal::Reserved {
14969 module_id: owner.to_string(),
14970 },
14971 scope_ref: scope_ref.to_string(),
14972 },
14973 )
14974 .await;
14975 assert_eq!(
14976 reply.header.ty,
14977 FrameType::Response,
14978 "{:?}",
14979 parse_error(&reply)
14980 );
14981 serde_json::from_slice(&reply.body).unwrap()
14982 }
14983
14984 #[tokio::test]
14985 async fn scope_describe_never_echoes_bind_proof_or_hash() {
14986 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
14987 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
14988 let mut rig = rig().await;
14989 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
14990 .await;
14991 let _bound = rig
14992 .bound(
14993 Some(AFT),
14994 PLEXUS,
14995 Some(
14996 rig.selector("s", Some(1))
14997 .with_bind_proof(Some(PROOF.into())),
14998 ),
14999 )
15000 .await;
15001 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
15002 let output = serde_json::to_string(&reply).unwrap();
15003 assert!(output.contains("\"status\":\"live\""));
15004 assert!(!output.contains(PROOF));
15005 assert!(!output.contains(HASH));
15006 }
15007
15008 async fn module(
15010 handler: &ControlHandler,
15011 connection: u64,
15012 module_id: &str,
15013 nonce: Option<&str>,
15014 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15015 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
15016 hello_via_sink(
15017 handler,
15018 &ctx,
15019 &mut rx,
15020 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
15021 )
15022 .await;
15023 (ctx, rx)
15024 }
15025
15026 #[tokio::test]
15029 async fn a_client_connection_cannot_sync_or_describe() {
15030 let handler = ControlHandler::new(Arc::new(Registry::default()));
15031 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
15032 for request in [
15033 ModuleControlRequestFromModule::ScopeSync {
15034 generation: 1,
15035 scopes: vec![head("s", 1)],
15036 },
15037 ModuleControlRequestFromModule::ScopeDescribe {
15038 owner: Principal::Direct,
15039 scope_ref: "s".to_string(),
15040 },
15041 ] {
15042 let reply = call(&handler, &ctx, &request).await;
15043 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
15044 }
15045 assert!(
15046 !handler
15047 .scopes
15048 .read()
15049 .unwrap()
15050 .describe(
15051 &Principal::Reserved {
15052 module_id: OWNER.to_string()
15053 },
15054 "s"
15055 )
15056 .owner_synced
15057 );
15058 }
15059
15060 #[tokio::test]
15063 async fn a_module_without_a_supervised_launch_cannot_sync() {
15064 let handler = ControlHandler::new(Arc::new(Registry::default()));
15065 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
15066 assert_eq!(
15067 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
15068 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15069 );
15070 }
15071
15072 #[tokio::test]
15073 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
15074 let supervisor = SupervisorHandle::new();
15075 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15076 let handler = ControlHandler::new(Arc::new(Registry::default()))
15077 .with_supervisor(supervisor.clone());
15078 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15079 sync(&handler, &incumbent, 1, vec![head("s", 1)])
15080 .await
15081 .expect("the current launch syncs");
15082
15083 supervisor.open_swap(OWNER, "n2".to_string());
15086 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
15087 assert_eq!(
15088 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15089 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15090 );
15091 sync(&handler, &incumbent, 2, vec![head("s", 1)])
15092 .await
15093 .expect("the serving owner syncs during the swap");
15094
15095 supervisor.close_swap(OWNER);
15098 assert_eq!(
15099 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15100 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15101 );
15102 sync(&handler, &incumbent, 3, vec![head("s", 1)])
15103 .await
15104 .expect("the serving owner syncs after the rollback");
15105 handler.cleanup_connection(candidate.connection_id).unwrap();
15106
15107 supervisor.open_swap(OWNER, "n3".to_string());
15112 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
15113 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
15114 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
15115 .await
15116 .expect("the promoted launch takes authority");
15117 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15118 panic!("unexpected reply {reply:?}");
15119 };
15120 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
15121 assert_eq!(
15122 sync(&handler, &incumbent, 4, Vec::new()).await,
15123 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15124 );
15125 }
15126
15127 #[tokio::test]
15130 async fn closing_the_authority_connection_frees_sync_authority() {
15131 let supervisor = SupervisorHandle::new();
15132 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15133 let handler = ControlHandler::new(Arc::new(Registry::default()))
15134 .with_supervisor(supervisor.clone());
15135 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15136 sync(&handler, &first, 10, vec![head("s", 1)])
15137 .await
15138 .unwrap();
15139 handler.cleanup_connection(first.connection_id).unwrap();
15140
15141 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
15142 sync(&handler, &second, 1, vec![head("s", 1)])
15143 .await
15144 .expect("the next connection takes the released authority");
15145 }
15146
15147 #[tokio::test]
15148 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
15149 let supervisor = SupervisorHandle::new();
15150 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15151 let handler =
15152 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
15153 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
15154 sync(&handler, &first, 10, vec![head("s", 1)])
15155 .await
15156 .unwrap();
15157 handler
15158 .handle_control_frame(
15159 &first,
15160 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
15161 )
15162 .await
15163 .unwrap();
15164 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
15165 sync(&handler, &second, 1, vec![head("s", 1)])
15166 .await
15167 .expect("GOODBYE releases authority even if the old socket remains open");
15168 }
15169
15170 #[tokio::test]
15171 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
15172 let registry = Arc::new(Registry::default());
15173 let supervisor_handle = SupervisorHandle::new();
15174 let supervisor =
15175 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15176 .with_handle(supervisor_handle.clone())
15177 .with_daemon_incarnation("incarnation-7".to_string());
15178 supervisor
15181 .supervise_configured(
15182 ModuleSpec {
15183 module_id: OWNER.to_string(),
15184 program: PathBuf::from("/nonexistent/prefrontal-core"),
15185 args: Vec::new(),
15186 env: Vec::new(),
15187 reserved: false,
15188 reserved_prefixes: Vec::new(),
15189 protocol: ModuleProtocol::Subc,
15190 overlap: Default::default(),
15191 },
15192 false,
15193 )
15194 .unwrap();
15195 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
15196 let handler =
15197 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15198 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
15199
15200 let ModuleControlResponseToModule::ScopeDescribe {
15202 status,
15203 daemon_incarnation,
15204 owner_synced,
15205 owner_configured,
15206 scope,
15207 ..
15208 } = describe(&handler, &reader, OWNER, "s").await
15209 else {
15210 panic!("not a describe reply");
15211 };
15212 assert_eq!(status, ScopeStatus::NotLive);
15213 assert_eq!(daemon_incarnation, "incarnation-7");
15214 assert!(!owner_synced);
15215 assert!(owner_configured);
15216 assert!(scope.is_none());
15217
15218 let ModuleControlResponseToModule::ScopeDescribe {
15221 status,
15222 owner_configured,
15223 ..
15224 } = describe(&handler, &reader, "ghost", "s").await
15225 else {
15226 panic!("not a describe reply");
15227 };
15228 assert_eq!(status, ScopeStatus::NotLive);
15229 assert!(!owner_configured);
15230
15231 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
15233 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
15234 let ModuleControlResponseToModule::ScopeDescribe {
15235 status,
15236 scope_epoch,
15237 owner_synced,
15238 scope,
15239 ..
15240 } = describe(&handler, &reader, OWNER, "s").await
15241 else {
15242 panic!("not a describe reply");
15243 };
15244 assert_eq!(status, ScopeStatus::Live);
15245 assert_eq!(scope_epoch, Some(4));
15246 assert!(owner_synced);
15247 let stamp = scope.expect("a live scope carries its stamp");
15248 assert!(
15249 stamp.owner_authorized,
15250 "prefrontal-core is the default authority"
15251 );
15252 assert_eq!(stamp.kind, ScopeKind::Head);
15253 }
15254
15255 #[tokio::test]
15256 async fn scope_authority_owners_decides_owner_authorized() {
15257 let supervisor = SupervisorHandle::new();
15258 supervisor.set_spawn_nonce("broca", "b1".to_string());
15259 let handler = ControlHandler::new(Arc::new(Registry::default()))
15260 .with_supervisor(supervisor)
15261 .with_scope_authority_owners(vec!["broca".to_string()]);
15262 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
15263 let mut gated = head("s", 1);
15264 gated.attributes.agent_id = Some("agent".to_string());
15265 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
15266 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
15267 describe(&handler, &broca, "broca", "s").await
15268 else {
15269 panic!("not a describe reply");
15270 };
15271 assert!(scope.unwrap().owner_authorized);
15272 }
15273
15274 #[tokio::test]
15278 async fn scope_ops_and_the_scopes_capability_are_advertised() {
15279 let handler = ControlHandler::new(Arc::new(Registry::default()));
15280 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
15281 let ack = hello_via_sink(
15282 &handler,
15283 &ctx,
15284 &mut rx,
15285 hello_frame("m", PROTOCOL_VERSION, 1),
15286 )
15287 .await;
15288 let ack = parse_ack(&ack);
15289 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
15290 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
15291 }
15292 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
15293
15294 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
15295 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15296 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15297 let reply = handler
15298 .handle_control_frame(&client, frame)
15299 .await
15300 .unwrap()
15301 .pop()
15302 .unwrap();
15303 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15304 serde_json::from_slice(&reply.body).unwrap()
15305 else {
15306 panic!("not a server.describe reply");
15307 };
15308 assert!(
15309 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
15310 "{capabilities:?}"
15311 );
15312 }
15313
15314 const PLEXUS: &str = "plexus";
15317 const OTHER: &str = "other";
15318 const AFT: &str = "aft";
15319 const BROCA: &str = "broca";
15320 const MAGIC: &str = "magic-context";
15321
15322 fn nonce(module_id: &str) -> String {
15323 format!("nonce-{module_id}")
15324 }
15325
15326 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15327 let (tx, rx) = mpsc::channel(64);
15328 (
15329 RouteCtx {
15330 connection_id: ConnectionId::new(connection),
15331 egress: FrameSink::new(tx),
15332 },
15333 rx,
15334 )
15335 }
15336
15337 struct Rig {
15341 handler: ControlHandler,
15342 forwarding: Arc<ForwardingTable>,
15343 owner: RouteCtx,
15344 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15345 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
15346 generation: u64,
15347 next_connection: u64,
15348 _supervisor: Supervisor,
15349 }
15350
15351 async fn rig() -> Rig {
15352 rig_with_flow_support(true).await
15353 }
15354
15355 async fn rig_with_flow_support(flow_support: bool) -> Rig {
15356 rig_with_scope_capabilities(flow_support, false).await
15357 }
15358
15359 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
15360 let mut provides = Vec::new();
15361 if flow_support {
15362 provides.push("flow-scopes/v1");
15363 }
15364 if run_support {
15365 provides.push("agent-run-scopes/v1");
15366 }
15367 rig_with_target_capabilities(&provides).await
15368 }
15369
15370 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
15371 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
15372 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15373 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15376 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
15377 Frame::build(
15378 FrameType::Hello,
15379 control_flags(),
15380 0,
15381 0,
15382 connection,
15383 serde_json::to_vec(&body).unwrap(),
15384 )
15385 .unwrap()
15386 }
15387
15388 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
15389 let registry = Arc::new(Registry::default());
15390 let forwarding = Arc::new(ForwardingTable::default());
15391 let supervisor_handle = SupervisorHandle::new();
15392 let supervisor =
15393 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15394 .with_handle(supervisor_handle.clone());
15395 supervisor
15396 .supervise_configured(
15397 ModuleSpec {
15398 module_id: OWNER.to_string(),
15399 program: PathBuf::from("/nonexistent/prefrontal-core"),
15400 args: Vec::new(),
15401 env: Vec::new(),
15402 reserved: false,
15403 reserved_prefixes: Vec::new(),
15404 protocol: ModuleProtocol::Subc,
15405 overlap: Default::default(),
15406 },
15407 false,
15408 )
15409 .unwrap();
15410 for module_id in [OWNER, AFT, BROCA, MAGIC] {
15411 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
15412 }
15413 let handler =
15414 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15415 .with_supervisor(supervisor_handle);
15416 let (owner, mut owner_rx) = wide_ctx(1);
15417 hello_via_sink(
15418 &handler,
15419 &owner,
15420 &mut owner_rx,
15421 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
15422 )
15423 .await;
15424 let mut modules = BTreeMap::new();
15425 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
15426 let (ctx, mut rx) = wide_ctx(connection);
15427 let hello = hello_with_capabilities(module_id, connection, provides);
15428 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
15429 modules.insert(module_id.to_string(), (ctx, rx));
15430 }
15431 Rig {
15432 handler,
15433 forwarding,
15434 owner,
15435 _owner_rx: owner_rx,
15436 modules,
15437 generation: 0,
15438 next_connection: 100,
15439 _supervisor: supervisor,
15440 }
15441 }
15442
15443 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
15444 ScopeCarrier::new(Principal::Reserved {
15445 module_id: module_id.to_string(),
15446 })
15447 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
15448 }
15449
15450 fn session(scope_epoch: u64) -> ScopeRecord {
15453 let mut record = head("s", scope_epoch);
15454 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
15455 record.attributes.agent_id = Some("agent-1".to_string());
15456 record.attributes.delegates = true;
15457 record
15458 }
15459
15460 impl Rig {
15461 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
15462 self.generation += 1;
15463 sync(&self.handler, &self.owner, self.generation, scopes)
15464 .await
15465 .expect("the owner's sync is accepted");
15466 }
15467
15468 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15469 ScopeSelector::new(
15470 Principal::Reserved {
15471 module_id: OWNER.to_string(),
15472 },
15473 scope_ref.to_string(),
15474 scope_epoch,
15475 )
15476 }
15477
15478 fn open_frame(
15479 &mut self,
15480 opener: Option<&str>,
15481 target: &str,
15482 scope: Option<ScopeSelector>,
15483 ) -> (
15484 RouteCtx,
15485 mpsc::Receiver<crate::router::OutboundFrame>,
15486 Frame,
15487 ) {
15488 self.next_connection += 1;
15489 let (ctx, rx) = wide_ctx(self.next_connection);
15490 let root = unique_project_root("scoped-open");
15491 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15492 target: RouteTarget::ToolProvider {
15493 module_id: target.to_string(),
15494 },
15495 identity: BindIdentity::new(
15496 root.path().to_path_buf(),
15497 "unit".to_string(),
15498 "session".to_string(),
15499 ),
15500 consumer_identity: opener.map(|module_id| ConsumerIdentity {
15501 module_id: module_id.to_string(),
15502 launch_nonce: nonce(module_id),
15503 }),
15504 consumer_capabilities: None,
15505 role_versions: None,
15506 admission_facts: None,
15507 scope,
15508 })
15509 .unwrap();
15510 let frame = Frame::build(
15511 FrameType::Request,
15512 control_flags(),
15513 0,
15514 0,
15515 self.next_connection,
15516 body,
15517 )
15518 .unwrap();
15519 (ctx, rx, frame)
15520 }
15521
15522 async fn refused(
15524 &mut self,
15525 opener: Option<&str>,
15526 target: &str,
15527 scope: Option<ScopeSelector>,
15528 ) -> String {
15529 self.refusal_body(opener, target, scope).await["code"]
15530 .as_str()
15531 .unwrap()
15532 .to_string()
15533 }
15534
15535 async fn refusal_body(
15536 &mut self,
15537 opener: Option<&str>,
15538 target: &str,
15539 scope: Option<ScopeSelector>,
15540 ) -> Value {
15541 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
15542 let replies = tokio::time::timeout(
15543 Duration::from_secs(2),
15544 self.handler.handle_control_frame(&ctx, frame),
15545 )
15546 .await
15547 .expect("the open must be refused before waiting for a bind ack")
15548 .unwrap();
15549 assert_eq!(replies.len(), 1, "{replies:?}");
15550 assert_eq!(replies[0].header.ty, FrameType::Error);
15551 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15552 assert!(
15553 module_rx.try_recv().is_err(),
15554 "a refused open relays nothing"
15555 );
15556 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
15557 parse_error(&replies[0])
15558 }
15559
15560 async fn relayed(
15562 &mut self,
15563 opener: Option<&str>,
15564 target: &str,
15565 scope: Option<ScopeSelector>,
15566 ) -> Relayed {
15567 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
15568 let handler = self.handler.clone();
15569 let task_ctx = ctx.clone();
15570 let task = tokio::spawn(async move {
15571 handler
15572 .handle_control_frame(&task_ctx, frame)
15573 .await
15574 .unwrap()
15575 });
15576 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15577 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
15578 .await
15579 .expect("the target receives the relayed route.bind")
15580 .unwrap()
15581 .frame;
15582 Relayed {
15583 target: target.to_string(),
15584 client: ctx,
15585 client_rx: rx,
15586 task,
15587 bind,
15588 }
15589 }
15590
15591 async fn ack(&self, relayed: &Relayed) {
15592 let (module, _) = &self.modules[&relayed.target];
15593 self.handler
15594 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
15595 .await
15596 .unwrap();
15597 }
15598
15599 async fn bound(
15601 &mut self,
15602 opener: Option<&str>,
15603 target: &str,
15604 scope: Option<ScopeSelector>,
15605 ) -> Bound {
15606 let relayed = self.relayed(opener, target, scope).await;
15607 self.ack(&relayed).await;
15608 let Relayed {
15609 target,
15610 client,
15611 mut client_rx,
15612 task,
15613 bind,
15614 } = relayed;
15615 assert!(
15616 task.await.unwrap().is_empty(),
15617 "the open is answered by commit"
15618 );
15619 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
15620 Bound {
15621 target,
15622 client,
15623 client_rx,
15624 channel,
15625 epoch,
15626 bind,
15627 }
15628 }
15629
15630 fn live(&self, route: &Bound) -> bool {
15631 matches!(
15632 self.forwarding
15633 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15634 .unwrap(),
15635 DataRoute::Client(DataRouteState::Bound(_))
15636 )
15637 }
15638 }
15639
15640 struct Relayed {
15641 target: String,
15642 client: RouteCtx,
15643 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15644 task: tokio::task::JoinHandle<Vec<Frame>>,
15645 bind: Frame,
15646 }
15647
15648 struct Bound {
15649 target: String,
15650 client: RouteCtx,
15651 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15652 channel: u16,
15653 epoch: u32,
15654 bind: Frame,
15655 }
15656
15657 impl Bound {
15658 fn closed_reason(&mut self) -> RouteCloseReason {
15661 let mut reason = None;
15662 let mut goodbye = false;
15663 while let Ok(outbound) = self.client_rx.try_recv() {
15664 let frame = outbound.frame;
15665 match frame.header.ty {
15666 FrameType::Goodbye => {
15667 assert_eq!(
15668 (frame.header.channel, frame.header.epoch),
15669 (self.channel, self.epoch)
15670 );
15671 goodbye = true;
15672 }
15673 FrameType::Push => {
15674 let ClientControlPush::RouteClosed {
15675 reason: r,
15676 module_id,
15677 ..
15678 } = serde_json::from_slice(&frame.body).unwrap()
15679 else {
15680 panic!("unexpected push");
15681 };
15682 assert_eq!(module_id, self.target);
15683 reason = Some(r);
15684 }
15685 other => panic!("unexpected frame {other:?}"),
15686 }
15687 }
15688 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
15689 reason.expect("the client is told why the route closed")
15690 }
15691
15692 fn untouched(&mut self) -> bool {
15693 self.client_rx.try_recv().is_err()
15694 }
15695
15696 fn stamp(&self) -> Option<ScopeStamp> {
15697 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
15698 ModuleControlRequest::RouteBind { scope, .. } => scope,
15699 other => panic!("expected a route.bind, got {other:?}"),
15700 }
15701 }
15702 }
15703
15704 #[tokio::test]
15705 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
15706 ) {
15707 let mut rig = rig().await;
15708 let mut record = session(1);
15709 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
15710 record.child_owners = vec![Principal::Reserved {
15711 module_id: MAGIC.to_string(),
15712 }];
15713 rig.sync(vec![record]).await;
15714 let scope = || Some(rig_selector("s", Some(1)));
15715
15716 rig.bound(Some(OWNER), PLEXUS, scope()).await;
15719 rig.bound(Some(AFT), OTHER, scope()).await;
15720 rig.bound(Some(BROCA), PLEXUS, scope()).await;
15721
15722 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
15726 assert_eq!(
15727 rig.refused(opener, target, scope()).await,
15728 error_codes::SCOPE_NOT_CARRIER,
15729 "{opener:?} -> {target}"
15730 );
15731 }
15732 }
15733
15734 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15735 ScopeSelector::new(
15736 Principal::Reserved {
15737 module_id: OWNER.to_string(),
15738 },
15739 scope_ref.to_string(),
15740 scope_epoch,
15741 )
15742 }
15743
15744 #[tokio::test]
15745 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
15746 let mut rig = rig().await;
15747 rig.sync(vec![session(1)]).await;
15748 for opener in [OWNER, AFT] {
15749 assert_eq!(
15750 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
15751 .await,
15752 error_codes::SCOPE_EPOCH_REQUIRED,
15753 "{opener}"
15754 );
15755 }
15756 }
15757
15758 #[tokio::test]
15759 async fn admission_separates_not_synced_not_live_and_ended() {
15760 let mut rig = rig().await;
15761 let code = rig
15763 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15764 .await;
15765 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
15766 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
15767
15768 let ghost = ScopeSelector::new(
15770 Principal::Reserved {
15771 module_id: "ghost".to_string(),
15772 },
15773 "s".to_string(),
15774 Some(1),
15775 );
15776 assert_eq!(
15777 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
15778 error_codes::SCOPE_NOT_LIVE
15779 );
15780
15781 rig.sync(vec![session(2)]).await;
15782 assert_eq!(
15783 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
15784 .await,
15785 error_codes::SCOPE_NOT_LIVE
15786 );
15787 for epoch in [1, 3] {
15788 assert_eq!(
15789 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
15790 .await,
15791 error_codes::SCOPE_ENDED,
15792 "epoch {epoch}"
15793 );
15794 }
15795 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
15797 .await;
15798 }
15799
15800 #[tokio::test]
15801 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
15802 let mut rig = rig().await;
15803 rig.sync(vec![session(1)]).await;
15804 let route = rig
15805 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15806 .await;
15807 let stamp = route.stamp().expect("a scoped bind carries the stamp");
15808 assert_eq!(stamp.scope_ref, "s");
15809 assert_eq!(stamp.scope_epoch, 1);
15810 assert_eq!(stamp.kind, ScopeKind::Head);
15811 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
15812 assert!(stamp.attributes.delegates);
15813 assert!(stamp.owner_authorized);
15814 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15815 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
15816
15817 let (broca, mut broca_rx) = wide_ctx(50);
15820 hello_via_sink(
15821 &rig.handler,
15822 &broca,
15823 &mut broca_rx,
15824 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
15825 )
15826 .await;
15827 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
15828 .await
15829 .unwrap();
15830 let own = ScopeSelector::new(
15831 Principal::Reserved {
15832 module_id: BROCA.to_string(),
15833 },
15834 "b".to_string(),
15835 Some(1),
15836 );
15837 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
15838 assert!(!route.stamp().unwrap().owner_authorized);
15839 }
15840
15841 #[tokio::test]
15842 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
15843 let mut rig = rig().await;
15844 let mut record = head("s", 1);
15845 record.carriers = vec![carrier(AFT, None)];
15846 let flow_id = "Flow:run-7/step_2!~";
15847 record.attributes.flow_id = Some(flow_id.to_string());
15848 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
15849 .await
15850 .unwrap();
15851 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15852 panic!("not a sync reply");
15853 };
15854 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
15855 let route = rig
15856 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15857 .await;
15858 assert!(rig.live(&route), "the stamped bind committed");
15859 let stamp = route.stamp().expect("a flow scope carries a stamp");
15860 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
15861 assert_eq!(stamp.attributes.agent_id, None);
15862 assert!(!stamp.attributes.delegates);
15863 assert!(stamp.owner_authorized);
15864 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15865 assert_eq!(unscoped.stamp(), None);
15866 }
15867
15868 #[tokio::test]
15869 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
15870 let mut rig = rig_with_flow_support(false).await;
15871 let mut record = session(1);
15872 record.attributes.flow_id = Some("flow:7".to_string());
15873 rig.sync(vec![record]).await;
15874 for opener in [OWNER, AFT] {
15875 let body = rig
15876 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15877 .await;
15878 assert_eq!(body["code"], "target_flow_unsupported");
15879 let message = body["message"].as_str().unwrap();
15880 for required in [PLEXUS, "flow-scopes/v1"] {
15881 assert!(message.contains(required), "{message}");
15882 }
15883 }
15884 }
15885
15886 #[tokio::test]
15887 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
15888 let mut rig = rig_with_flow_support(true).await;
15889 let mut record = session(1);
15890 record.attributes.flow_id = Some("flow:7".to_string());
15891 rig.sync(vec![record]).await;
15892 for opener in [OWNER, AFT] {
15893 let route = rig
15894 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15895 .await;
15896 assert!(rig.live(&route));
15897 assert_eq!(
15898 route.stamp().unwrap().attributes.flow_id.as_deref(),
15899 Some("flow:7")
15900 );
15901 }
15902 }
15903
15904 #[tokio::test]
15905 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
15906 use std::future::Future;
15907
15908 let mut rig = rig_with_flow_support(true).await;
15909 let mut record = session(1);
15910 record.attributes.flow_id = Some("flow:7".to_string());
15911 rig.sync(vec![record]).await;
15912 let (client, mut client_rx, frame) =
15913 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
15914 for _ in 0..64 {
15917 client.egress.try_send(route_bind_ack(1)).unwrap();
15918 }
15919 let handler = rig.handler.clone();
15920 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15921 std::future::poll_fn(|cx| {
15922 assert!(open.as_mut().poll(cx).is_pending());
15923 std::task::Poll::Ready(())
15924 })
15925 .await;
15926 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15927
15928 let old_connection = rig.modules[PLEXUS].0.connection_id;
15929 rig.handler.cleanup_connection(old_connection).unwrap();
15930 let (replacement, mut replacement_rx) = wide_ctx(200);
15931 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
15932 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15933 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15934 let hello = Frame::build(
15935 FrameType::Hello,
15936 control_flags(),
15937 0,
15938 0,
15939 200,
15940 serde_json::to_vec(&body).unwrap(),
15941 )
15942 .unwrap();
15943 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
15944
15945 client_rx.try_recv().unwrap();
15946 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15947 .await
15948 .expect("the replacement is refused without waiting for a bind ack")
15949 .unwrap();
15950 assert_eq!(replies.len(), 1);
15951 let body = parse_error(&replies[0]);
15952 assert_eq!(body["code"], "target_flow_unsupported");
15953 for required in [PLEXUS, "flow-scopes/v1"] {
15954 assert!(body["message"].as_str().unwrap().contains(required));
15955 }
15956 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
15957 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15958 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15959 assert!(rig
15960 .handler
15961 .registry
15962 .get_module_by_connection(replacement.connection_id)
15963 .unwrap()
15964 .is_some());
15965 }
15966
15967 #[tokio::test]
15968 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
15969 use std::future::Future;
15970 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15971 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15972 const NEXT_HASH: &str =
15973 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
15974 let mut rig = rig().await;
15975 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
15976 record.carriers.push(ScopeCarrier::new(Principal::Direct));
15977 rig.sync(vec![record.clone()]).await;
15978 let (client, mut rx, frame) = rig.open_frame(
15979 None,
15980 PLEXUS,
15981 Some(
15982 rig.selector("s", Some(1))
15983 .with_bind_proof(Some(PROOF.into())),
15984 ),
15985 );
15986 for _ in 0..64 {
15989 client.egress.try_send(route_bind_ack(1)).unwrap();
15990 }
15991 let handler = rig.handler.clone();
15992 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15993 std::future::poll_fn(|cx| {
15994 assert!(open.as_mut().poll(cx).is_pending());
15995 std::task::Poll::Ready(())
15996 })
15997 .await;
15998 record.bind_proof_sha256 = Some(NEXT_HASH.into());
15999 rig.sync(vec![record]).await;
16000 rx.try_recv().unwrap();
16001 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16002 .await
16003 .unwrap()
16004 .unwrap();
16005 assert_eq!(
16006 parse_error(&replies[0])["code"],
16007 error_codes::SCOPE_BIND_PROOF_MISMATCH
16008 );
16009 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
16010 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16011 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16012 }
16013
16014 #[tokio::test]
16015 async fn scope_bind_proof_reserved_openers_ignore_proof() {
16016 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16017 let mut rig = rig().await;
16018 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16019 .await;
16020 for opener in [OWNER, AFT] {
16021 for proof in [None, Some("malformed+proof=")] {
16022 let route = rig
16023 .bound(
16024 Some(opener),
16025 PLEXUS,
16026 Some(
16027 rig.selector("s", Some(1))
16028 .with_bind_proof(proof.map(str::to_string)),
16029 ),
16030 )
16031 .await;
16032 assert!(rig.live(&route));
16033 }
16034 }
16035 }
16036
16037 #[tokio::test]
16038 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
16039 use std::future::Future;
16040 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16041 const NEXT_HASH: &str =
16042 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16043 let mut rig = rig().await;
16044 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16045 .await;
16046 let (client, mut rx, frame) =
16047 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
16048 for _ in 0..64 {
16049 client.egress.try_send(route_bind_ack(1)).unwrap();
16050 }
16051 let handler = rig.handler.clone();
16052 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16053 std::future::poll_fn(|cx| {
16054 assert!(open.as_mut().poll(cx).is_pending());
16055 std::task::Poll::Ready(())
16056 })
16057 .await;
16058 rig.sync(vec![
16059 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
16060 ])
16061 .await;
16062 rx.try_recv().unwrap();
16063 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16064 let bind = std::future::poll_fn(|cx| {
16065 assert!(open.as_mut().poll(cx).is_pending());
16066 module_rx.poll_recv(cx)
16067 })
16068 .await
16069 .unwrap()
16070 .frame;
16071 handler
16072 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
16073 .await
16074 .unwrap();
16075 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16076 .await
16077 .unwrap()
16078 .unwrap();
16079 assert_eq!(replies.len(), 1);
16080 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
16083 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16084 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
16085 let route = rig
16086 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
16087 .await;
16088 assert!(rig.live(&route));
16089 }
16090
16091 #[tokio::test]
16092 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
16093 {
16094 let mut rig = rig_with_flow_support(false).await;
16095 rig.sync(vec![session(1)]).await;
16096 let route = rig
16097 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16098 .await;
16099 assert!(rig.live(&route));
16100 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
16101 let mut record = session(1);
16102 record.attributes.flow_id = Some("flow:7".to_string());
16103 rig.sync(vec![record]).await;
16104 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
16105 assert!(rig.live(&unscoped));
16106 assert_eq!(unscoped.stamp(), None);
16107 }
16108
16109 #[tokio::test]
16110 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
16111 let mut rig = rig().await;
16112 let mut record = session(1);
16113 record.attributes.flow_id = Some("flow:7".to_string());
16114 rig.sync(vec![record.clone()]).await;
16115 let before = rig.forwarding.published_scope_tag(OWNER, "s");
16116 let mut owner_route = rig
16117 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16118 .await;
16119 let mut carrier_route = rig
16120 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
16121 .await;
16122 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16123 rig.sync(vec![record.clone()]).await;
16124 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16125 assert!(rig.live(&owner_route) && owner_route.untouched());
16126 assert!(rig.live(&carrier_route) && carrier_route.untouched());
16127
16128 record.attributes.flow_id = Some("flow:8".to_string());
16129 rig.sync(vec![record]).await;
16130 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
16131 let before = before.unwrap();
16132 assert_eq!(after.scope_epoch, before.scope_epoch);
16133 assert!(after.version > before.version);
16134 for route in [&mut owner_route, &mut carrier_route] {
16135 assert!(!rig.live(route));
16136 assert_eq!(
16137 route.closed_reason(),
16138 RouteCloseReason::ScopeDelegationChanged
16139 );
16140 }
16141 assert!(rig.live(&unscoped) && unscoped.untouched());
16142 for target in [PLEXUS, OTHER] {
16145 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
16146 let goodbye = module_rx
16147 .try_recv()
16148 .expect("the provider sees the drain")
16149 .frame;
16150 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16151 assert!(module_rx.try_recv().is_err());
16152 }
16153 let rebound = rig
16154 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16155 .await;
16156 assert_eq!(
16157 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
16158 Some("flow:8")
16159 );
16160 }
16161
16162 #[tokio::test]
16167 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
16168 let mut rig = rig().await;
16169 rig.sync(vec![session(1)]).await;
16170 let mut cotenant = rig
16171 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16172 .await;
16173
16174 let mut changed = session(1);
16175 changed.child_owners.push(Principal::Reserved {
16176 module_id: MAGIC.to_string(),
16177 });
16178 let mut ended = None;
16179 for (code, next) in [
16180 (error_codes::SCOPE_CHANGED, vec![changed]),
16181 (error_codes::SCOPE_ENDED, Vec::new()),
16182 ] {
16183 let relayed = rig
16184 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16185 .await;
16186 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
16187 ended = Some(next.is_empty());
16188 rig.sync(next).await;
16189 rig.ack(&relayed).await;
16190 let replies = relayed.task.await.unwrap();
16191 assert_eq!(replies.len(), 1, "{replies:?}");
16192 assert_eq!(parse_error(&replies[0])["code"], code);
16193 assert_eq!(
16194 subc_protocol::error_codes::is_retryable_route_open(code),
16195 code == error_codes::SCOPE_CHANGED
16196 );
16197 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16198 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16200 let mut goodbyes = Vec::new();
16203 while let Ok(outbound) = plexus_rx.try_recv() {
16204 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
16205 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
16206 }
16207 assert!(
16208 goodbyes.contains(&(bind_channel, bind_epoch)),
16209 "{goodbyes:?}"
16210 );
16211 assert!(rig
16212 .handler
16213 .registry
16214 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
16215 .unwrap()
16216 .is_some());
16217 }
16218 assert_eq!(ended, Some(true));
16219 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
16222 }
16223
16224 #[tokio::test]
16227 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
16228 struct Case {
16229 name: &'static str,
16230 change: fn(&mut ScopeRecord),
16231 closed: [Option<RouteCloseReason>; 4],
16234 }
16235 use RouteCloseReason::*;
16236 let cases = [
16237 Case {
16238 name: "a carrier entry removed",
16239 change: |r| {
16240 r.carriers.retain(|c| {
16241 c.principal
16242 != Principal::Reserved {
16243 module_id: AFT.to_string(),
16244 }
16245 })
16246 },
16247 closed: [None, Some(ScopeCarrierRemoved), None, None],
16248 },
16249 Case {
16250 name: "a target removed from a carrier",
16251 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
16252 closed: [None, None, None, Some(ScopeCarrierRemoved)],
16253 },
16254 Case {
16255 name: "a bare carrier narrowed to targets",
16256 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
16257 closed: [None, Some(ScopeCarrierRemoved), None, None],
16258 },
16259 Case {
16260 name: "delegates turned off",
16261 change: |r| r.attributes.delegates = false,
16262 closed: [Some(ScopeDelegationChanged); 4],
16263 },
16264 Case {
16265 name: "agent_id changed",
16266 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
16267 closed: [Some(ScopeDelegationChanged); 4],
16268 },
16269 Case {
16270 name: "a carrier added, child owners changed, the record re-sent",
16271 change: |r| {
16272 r.carriers.push(carrier(MAGIC, None));
16273 r.child_owners.push(Principal::Reserved {
16274 module_id: MAGIC.to_string(),
16275 });
16276 },
16277 closed: [None; 4],
16278 },
16279 Case {
16280 name: "a target added",
16281 change: |r| {
16282 r.carriers[1]
16283 .targets
16284 .as_mut()
16285 .unwrap()
16286 .push("third".to_string())
16287 },
16288 closed: [None; 4],
16289 },
16290 Case {
16291 name: "delegates turned on",
16292 change: |r| r.attributes.delegates = true,
16293 closed: [None; 4],
16294 },
16295 ];
16296 for case in cases {
16297 let mut rig = rig().await;
16298 rig.sync(vec![session(1)]).await;
16299 let scope = || Some(rig_selector("s", Some(1)));
16300 let mut routes = [
16301 rig.bound(Some(OWNER), PLEXUS, scope()).await,
16302 rig.bound(Some(AFT), PLEXUS, scope()).await,
16303 rig.bound(Some(BROCA), PLEXUS, scope()).await,
16304 rig.bound(Some(BROCA), OTHER, scope()).await,
16305 ];
16306 let mut record = session(1);
16307 (case.change)(&mut record);
16308 rig.sync(vec![record]).await;
16309 for (index, expected) in case.closed.iter().enumerate() {
16310 let route = &mut routes[index];
16311 match expected {
16312 Some(reason) => {
16313 assert!(!rig.live(route), "{}: route {index} still live", case.name);
16314 assert_eq!(
16315 route.closed_reason(),
16316 *reason,
16317 "{}: route {index}",
16318 case.name
16319 );
16320 }
16321 None => {
16322 assert!(rig.live(route), "{}: route {index} closed", case.name);
16323 assert!(
16324 route.untouched(),
16325 "{}: route {index} was told something",
16326 case.name
16327 );
16328 }
16329 }
16330 }
16331 }
16332 }
16333
16334 #[tokio::test]
16335 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
16336 for next in [Vec::new(), vec![session(2)]] {
16338 let mut rig = rig().await;
16339 rig.sync(vec![session(1)]).await;
16340 let mut route = rig
16341 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16342 .await;
16343 rig.sync(next).await;
16344 assert!(!rig.live(&route));
16345 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
16346 }
16347
16348 let mut rig = rig().await;
16351 let mut child = session(1);
16352 child.scope_ref = "child".to_string();
16353 child.kind = ScopeKind::Worker;
16354 child.parent = Some(ScopeParent::new(
16355 Principal::Reserved {
16356 module_id: OWNER.to_string(),
16357 },
16358 "s".to_string(),
16359 1,
16360 ));
16361 rig.sync(vec![session(1), child.clone()]).await;
16362 let mut child_route = rig
16363 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
16364 .await;
16365 assert_eq!(
16366 child_route.stamp().unwrap().parent_state,
16367 Some(ParentState::Linked)
16368 );
16369 rig.sync(vec![child]).await;
16370 assert!(!rig.live(&child_route));
16371 assert_eq!(
16372 child_route.closed_reason(),
16373 RouteCloseReason::ScopeParentEnded
16374 );
16375 }
16376
16377 #[tokio::test]
16378 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
16379 ) {
16380 let mut rig = rig().await;
16381 rig.sync(vec![session(1)]).await;
16382 let mut route = rig
16383 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16384 .await;
16385 let before = rig.forwarding.published_scope_tag(OWNER, "s");
16386 rig.sync(vec![session(1)]).await;
16387 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16388 assert!(rig.live(&route) && route.untouched());
16389
16390 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
16394 .forwarding
16395 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16396 .unwrap()
16397 else {
16398 panic!("the route is bound");
16399 };
16400 binding.flow.acquire_tagged(9, false).await.unwrap();
16401 let mut widened = session(1);
16402 widened.carriers.push(carrier(MAGIC, None));
16403 rig.sync(vec![widened]).await;
16404 assert!(rig.live(&route) && route.untouched());
16405 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16406 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
16407 assert_eq!(binding.flow.in_flight(), 1);
16410 binding
16411 .flow
16412 .acquire_tagged(10, false)
16413 .await
16414 .expect("the flow is still open");
16415 }
16416
16417 #[tokio::test]
16420 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
16421 let mut rig = rig().await;
16422 rig.sync(vec![session(1)]).await;
16423 let mut on_incumbent = rig
16424 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16425 .await;
16426
16427 let (candidate, _candidate_rx) = wide_ctx(9);
16430 let registration = rig
16431 .handler
16432 .registry
16433 .register_candidate_with_control_ops(
16434 manifest(PLEXUS, PROTOCOL_VERSION),
16435 PROTOCOL_VERSION,
16436 candidate.connection_id,
16437 module_baseline_control_ops(),
16438 )
16439 .unwrap();
16440 rig.forwarding
16441 .register_candidate_module_connection(
16442 candidate.connection_id,
16443 PLEXUS.to_string(),
16444 PROTOCOL_VERSION,
16445 manifest_concurrency(®istration.manifest),
16446 candidate.egress.clone(),
16447 )
16448 .unwrap();
16449 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
16450 rig.handler
16451 .registry
16452 .promote_candidate(PLEXUS)
16453 .unwrap()
16454 .unwrap();
16455 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
16456
16457 rig.sync(Vec::new()).await;
16458 assert!(!rig.live(&on_incumbent));
16459 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
16460 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16461 let goodbye = incumbent_rx
16462 .try_recv()
16463 .expect("the superseded endpoint is told")
16464 .frame;
16465 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16466 }
16467 }
16468}
16469
16470#[cfg(test)]
16471mod concurrency_default_exposure_tests {
16472 use super::*;
16473
16474 fn hello_body(role_json: &str) -> Vec<u8> {
16475 format!(
16476 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
16477 )
16478 .into_bytes()
16479 }
16480
16481 fn manifest_from(body: &[u8]) -> ModuleManifest {
16482 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
16483 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
16484 .expect("manifest parses")
16485 }
16486
16487 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
16488
16489 #[test]
16490 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
16491 let body = hello_body(&format!(
16492 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
16493 ));
16494 let manifest = manifest_from(&body);
16495 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16498 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
16499 }
16500
16501 #[test]
16502 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
16503 let body = hello_body(&format!(
16504 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
16505 ));
16506 let manifest = manifest_from(&body);
16507 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16508 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16509 }
16510
16511 #[test]
16512 fn non_management_roles_are_never_reported() {
16513 let body = hello_body(
16514 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
16515 );
16516 let manifest = manifest_from(&body);
16517 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16518 }
16519}