1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
26 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
27 },
28 scope::{
29 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
30 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
31 SCOPE_SYNC_OP,
32 },
33 session::{
34 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
35 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
36 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
37 ROLE_VERSIONS_FIELD,
38 },
39 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
40 Priority, RouteTarget, PROTOCOL_VERSION,
41};
42use tokio::time::{timeout_at, Instant};
43use tracing::{debug, info, warn};
44
45use crate::{
46 caller_identity::CallerIdentity,
47 capability_requirements::{
48 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
49 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
50 RuntimeModule,
51 },
52 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
53 forwarding::{
54 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
55 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
56 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
57 },
58 observability::{
59 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
60 },
61 privileged_prompt::{prompt_text, PrivilegedOperation},
62 provenance::{
63 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
64 },
65 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
66 router::{RouteCtx, RouterError},
67 scopes::{HelloLaunchNonces, ScopeTable},
68 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
69 stderr_tail::{CaptureState, TailEntry},
70 supervise::{
71 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
72 SupervisorHandle, SwapHelloAdmission,
73 },
74 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
75};
76
77pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
83
84const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
85const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
86const CAP_PING_PONG: &str = "ping_pong_v1";
87const CAP_SESSION_ATTACH: &str = "session_attach_v1";
88const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
89
90const SUBC_CONTROL_OPS: &[&str] = &[
91 ops::ACTIVITY_RECORD,
92 ops::ACTIVITY_QUERY,
93 ops::ACTIVITY_SUBSCRIBE,
94 ops::SERVER_DESCRIBE,
95 ops::CATALOG_LIST,
96 ops::ROUTE_OPEN,
97 ops::ROUTE_POLL,
98 ops::ROUTE_CLOSING,
99 ops::ROUTE_CLOSED,
100 ops::SUPERVISOR_LIST,
101 ops::SUPERVISOR_RESTART,
102 ops::SUPERVISOR_SWAP,
103 ops::SUPERVISOR_RELOAD,
104 ops::SUPERVISOR_RESCAN,
105 ops::SUPERVISOR_RELEASE_RESERVED,
106 ops::SUPERVISOR_SET_ENABLED,
107 ops::SUPERVISOR_HEALTH_PROBE,
108 ops::SUPERVISOR_HEALTH,
109 ops::SUPERVISOR_STDERR_TAIL,
110 ops::SUPERVISOR_TERMINALS,
111 ops::SUPERVISOR_ROUTES,
112 ops::SUPERVISOR_PROVENANCE,
113 ops::SUPERVISOR_SPAWN_SNAPSHOT,
114 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
115];
116
117const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
118 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
119 "supervisor.live_roots",
120 SCOPE_SYNC_OP,
121 SCOPE_APPLY_OP,
122 SCOPE_DESCRIBE_OP,
123 "operator.confirm",
124];
125
126const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
130
131const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
132
133pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
141
142pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
157
158pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
176
177const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
178const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
179
180fn reload_verdict(
181 configured: &Path,
182 spawned_from: Option<&Path>,
183 image: subc_control::RunningImageAgreement,
184) -> PendingReloadVerdict {
185 let path = match spawned_from {
186 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
187 ReloadPathAgreement::Match
188 }
189 Some(spawned_from) => ReloadPathAgreement::Mismatch {
190 configured: configured.to_path_buf(),
191 spawned_from: spawned_from.to_path_buf(),
192 },
193 None => ReloadPathAgreement::Unavailable {
194 reason: if matches!(
195 image,
196 subc_control::RunningImageAgreement::Unavailable {
197 reason: subc_control::RunningImageUnavailableReason::NotRunning
198 }
199 ) {
200 ReloadPathUnavailableReason::NotRunning
201 } else {
202 ReloadPathUnavailableReason::SpawnedPathUnavailable
203 },
204 },
205 };
206 PendingReloadVerdict {
207 path,
208 image,
209 spec: None,
210 }
211}
212
213fn launch_spec_agreement(
214 configured: &crate::supervise::ModuleSpec,
215 running: bool,
216 spawned: Option<&crate::supervise::LaunchSpecDigest>,
217) -> subc_control::LaunchSpecAgreement {
218 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
219 if !running {
220 return Agreement::Unavailable {
221 reason: Reason::NotRunning,
222 };
223 }
224 match spawned {
225 None => Agreement::Unavailable {
226 reason: Reason::SpawnedSpecUnavailable,
227 },
228 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
229 Agreement::Match
230 }
231 Some(_) => Agreement::Mismatch,
232 }
233}
234
235fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
236 if configured == spawned_from {
237 return true;
238 }
239 #[cfg(windows)]
240 {
241 match (
247 std::fs::canonicalize(configured),
248 std::fs::canonicalize(spawned_from),
249 ) {
250 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
251 _ => false,
252 }
253 }
254 #[cfg(not(windows))]
255 {
256 false
257 }
258}
259
260#[derive(Clone)]
261struct DaemonProvenanceFacts {
262 build: DaemonBuildProvenance,
263 pid: Option<u32>,
264 started_at_ms: Option<u64>,
265 start_clock: Option<crate::clock::StartClock>,
266 executable_path: Option<PathBuf>,
267 executable_identity: Option<SpawnedFileIdentity>,
268 process_start_time: Option<u64>,
269 probe: ExecutableIdentityProbe,
270}
271
272impl Default for DaemonProvenanceFacts {
273 fn default() -> Self {
274 Self {
275 build: DaemonBuildProvenance {
276 build_git_sha: None,
277 build_lock_digest: None,
278 },
279 pid: None,
280 started_at_ms: None,
281 start_clock: None,
282 executable_path: None,
283 executable_identity: None,
284 process_start_time: None,
285 probe: ExecutableIdentityProbe::default(),
286 }
287 }
288}
289
290#[derive(Debug, Clone)]
291struct SupervisorRescanContext {
292 supervisor: Supervisor,
293 config_path: PathBuf,
294 configured_port: Option<u16>,
295 storage_config: Option<crate::daemon_config::StorageConfig>,
296 admission_facts_carrier_module_id: Option<String>,
297 admission_facts_targets: Option<Vec<String>>,
298 scope_authority_owners: Vec<String>,
299 privileged_operator_prompt: PrivilegedOperatorPrompt,
300}
301
302const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
323 "reloading",
324 "supervisor_not_live",
325 "registration_not_active",
326 "no_forwarding_connection",
327 "relay_send_failed",
328];
329
330#[derive(Clone)]
332pub struct ControlHandler {
333 registry: Arc<Registry>,
334 activity: Option<crate::activity::ActivityLedger>,
335 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
336 forwarding: Arc<ForwardingTable>,
337 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
338 supervisor: SupervisorHandle,
339 subc_capabilities: Arc<[String]>,
340 route_bind_relay_timeout: Duration,
344 route_bind_relay_timeouts: BTreeMap<String, Duration>,
348 route_bind_breakers: RouteBindBreakers,
351 route_bind_concurrency: RouteBindConcurrency,
354 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
359 route_bind_breaker_threshold: u32,
361 route_bind_breaker_cooldown: Duration,
363 health_probe_timeout: Duration,
364 storage_config: Option<crate::daemon_config::StorageConfig>,
367 machine_id: Option<crate::machine_id::MachineId>,
371 admission_facts_carrier_module_id: Option<String>,
372 admission_facts_targets: Option<Vec<String>>,
373 scopes: Arc<RwLock<ScopeTable>>,
377 scope_deadline_added: Arc<tokio::sync::Notify>,
381 scope_authority_owners: Vec<String>,
384 privileged_operator_prompt: PrivilegedOperatorPrompt,
386 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
389 #[cfg(test)]
390 caller_lookup: Option<TestCallerLookup>,
391 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
394 rescan: Option<SupervisorRescanContext>,
395 connected_clients: ConnectedClients,
396 counters: DaemonCounters,
397 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
398 daemon_provenance: DaemonProvenanceFacts,
399 #[cfg(test)]
400 control_dispatch_delay: Option<Duration>,
401 #[cfg(test)]
402 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
403 #[cfg(test)]
407 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
408 #[cfg(test)]
414 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
415}
416
417impl fmt::Debug for ControlHandler {
418 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
419 f.debug_struct("ControlHandler")
420 .field("registry", &self.registry)
421 .field("forwarding", &self.forwarding)
422 .field("process_liveness", &self.process_liveness.is_some())
423 .field("supervisor", &self.supervisor)
424 .field("subc_capabilities", &self.subc_capabilities)
425 .finish()
426 }
427}
428
429struct RouteOpenRequest {
430 target: RouteTarget,
431 identity: BindIdentity,
432 consumer_identity: Option<ConsumerIdentity>,
433 consumer_capabilities: Option<Vec<String>>,
434 role_versions: Option<BTreeMap<String, String>>,
435 admission_facts: Option<serde_json::Value>,
436 scope: Option<ScopeSelector>,
437}
438
439#[derive(Clone, Copy)]
440enum RouteOpenPhase {
441 Admission,
442 ProjectRoot,
443 EgressReservation,
444 BindSend,
445 BindAckWait,
446}
447
448struct RouteOpenTimings {
452 module_id: String,
453 principal: String,
454 admission: Duration,
455 project_root: Duration,
456 egress_reservation: Duration,
457 bind_send: Duration,
458 bind_ack_wait: Duration,
459 phase: Option<RouteOpenPhase>,
460 phase_started_at: Instant,
461}
462
463impl RouteOpenTimings {
464 fn new() -> Self {
465 Self {
466 module_id: String::new(),
467 principal: "unresolved".to_string(),
469 admission: Duration::ZERO,
470 project_root: Duration::ZERO,
471 egress_reservation: Duration::ZERO,
472 bind_send: Duration::ZERO,
473 bind_ack_wait: Duration::ZERO,
474 phase: Some(RouteOpenPhase::Admission),
475 phase_started_at: Instant::now(),
476 }
477 }
478
479 fn finish(&mut self) {
480 let elapsed = self.phase_started_at.elapsed();
481 match self.phase.take() {
482 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
483 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
484 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
485 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
486 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
487 None => {}
488 }
489 }
490
491 fn enter(&mut self, phase: RouteOpenPhase) {
492 self.finish();
493 self.phase = Some(phase);
494 self.phase_started_at = Instant::now();
495 }
496}
497
498struct RouteBindReservationGuard {
499 forwarding: Arc<ForwardingTable>,
500 endpoint: ModuleEndpointId,
501 relay_corr: u64,
502 armed: bool,
503}
504
505struct ModuleControlRpcGuard {
506 forwarding: Arc<ForwardingTable>,
507 endpoint: ModuleEndpointId,
508 corr: u64,
509 armed: bool,
510}
511
512impl ModuleControlRpcGuard {
513 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
514 Self {
515 forwarding,
516 endpoint,
517 corr,
518 armed: true,
519 }
520 }
521
522 fn disarm(&mut self) {
523 self.armed = false;
524 }
525}
526
527impl Drop for ModuleControlRpcGuard {
528 fn drop(&mut self) {
529 if self.armed {
530 let _ = self
531 .forwarding
532 .cancel_module_control_rpc(self.endpoint, self.corr);
533 }
534 }
535}
536
537impl RouteBindReservationGuard {
538 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
539 Self {
540 forwarding,
541 endpoint,
542 relay_corr,
543 armed: true,
544 }
545 }
546
547 fn release_and_disarm(&mut self) {
548 if !self.armed {
549 return;
550 }
551 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
552 self.endpoint,
553 self.relay_corr,
554 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
555 ) {
556 send_goodbye_target_best_effort(
557 &self.forwarding.counters(),
558 &target,
559 "canceled route.bind",
560 );
561 }
562 self.armed = false;
563 }
564
565 fn disarm(&mut self) {
566 self.armed = false;
567 }
568}
569
570impl Drop for RouteBindReservationGuard {
571 fn drop(&mut self) {
572 self.release_and_disarm();
573 }
574}
575
576#[derive(Debug, Clone, Default)]
606pub(crate) struct RouteBindBreakers {
607 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
608}
609
610#[derive(Debug, Clone, Default)]
611pub(crate) struct RouteBindConcurrency {
612 modules: Arc<Mutex<HashMap<String, usize>>>,
613}
614
615struct RouteBindConcurrencyGuard {
616 concurrency: RouteBindConcurrency,
617 module_id: String,
618}
619
620impl RouteBindConcurrency {
621 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
624 let mut modules = self
625 .modules
626 .lock()
627 .expect("route.bind concurrency mutex poisoned");
628 let in_flight = modules.entry(module_id.to_string()).or_default();
629 if *in_flight >= limit {
630 return Err(*in_flight);
631 }
632 *in_flight += 1;
633 Ok(RouteBindConcurrencyGuard {
634 concurrency: self.clone(),
635 module_id: module_id.to_string(),
636 })
637 }
638}
639
640impl Drop for RouteBindConcurrencyGuard {
641 fn drop(&mut self) {
642 let mut modules = self
643 .concurrency
644 .modules
645 .lock()
646 .expect("route.bind concurrency mutex poisoned");
647 let remove = {
648 let in_flight = modules
649 .get_mut(&self.module_id)
650 .expect("admitted route.bind has a concurrency entry");
651 *in_flight -= 1;
652 *in_flight == 0
653 };
654 if remove {
655 modules.remove(&self.module_id);
656 }
657 }
658}
659
660#[derive(Debug, Default)]
661struct ModuleBreakerState {
662 consecutive_timeouts: u32,
664 cooldown_until: Option<Instant>,
667 probe_in_flight: Option<Arc<()>>,
672}
673
674enum RouteBindAdmission<'a> {
676 Admitted {
677 guard: RouteBindBreakerGuard<'a>,
678 probe: bool,
681 },
682 Refused {
683 consecutive_timeouts: u32,
684 retry_in: Duration,
688 probe_in_flight: bool,
689 },
690}
691
692struct RouteBindBreakerGuard<'a> {
699 breakers: RouteBindBreakers,
700 module_id: &'a str,
701 probe_token: Option<Arc<()>>,
702 settled: bool,
703}
704
705impl RouteBindBreakerGuard<'_> {
706 fn record_accepted(&mut self) -> bool {
710 self.settled = true;
711 self.breakers.record_accepted(self.module_id)
712 }
713
714 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
717 self.settled = true;
718 self.breakers.record_timeout(
719 self.module_id,
720 self.probe_token.as_ref(),
721 threshold,
722 cooldown,
723 )
724 }
725
726 fn record_inconclusive(&mut self) {
735 self.settled = true;
736 self.breakers
737 .record_inconclusive(self.module_id, self.probe_token.as_ref());
738 }
739}
740
741impl Drop for RouteBindBreakerGuard<'_> {
742 fn drop(&mut self) {
743 if !self.settled {
744 self.breakers
745 .record_inconclusive(self.module_id, self.probe_token.as_ref());
746 }
747 }
748}
749
750struct BreakerOpened {
754 consecutive_timeouts: u32,
755 reopened_after_probe: bool,
758}
759
760impl RouteBindBreakers {
761 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
762 self.modules
763 .lock()
764 .expect("route.bind breaker mutex poisoned")
765 }
766
767 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
770 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
771 probe: probe_token.is_some(),
772 guard: RouteBindBreakerGuard {
773 breakers: self.clone(),
774 module_id,
775 probe_token,
776 settled: false,
777 },
778 };
779
780 let mut modules = self.lock();
781 let Some(state) = modules.get_mut(module_id) else {
782 return admitted(None);
783 };
784 let Some(cooldown_until) = state.cooldown_until else {
785 return admitted(None);
786 };
787 if state.probe_in_flight.is_some() {
788 return RouteBindAdmission::Refused {
789 consecutive_timeouts: state.consecutive_timeouts,
790 retry_in: Duration::ZERO,
791 probe_in_flight: true,
792 };
793 }
794 let now = Instant::now();
795 if now < cooldown_until {
796 return RouteBindAdmission::Refused {
797 consecutive_timeouts: state.consecutive_timeouts,
798 retry_in: cooldown_until - now,
799 probe_in_flight: false,
800 };
801 }
802 let token = Arc::new(());
803 state.probe_in_flight = Some(Arc::clone(&token));
804 admitted(Some(token))
805 }
806
807 fn record_accepted(&self, module_id: &str) -> bool {
808 self.lock()
809 .remove(module_id)
810 .is_some_and(|state| state.cooldown_until.is_some())
811 }
812
813 fn record_timeout(
814 &self,
815 module_id: &str,
816 probe_token: Option<&Arc<()>>,
817 threshold: u32,
818 cooldown: Duration,
819 ) -> Option<BreakerOpened> {
820 let mut modules = self.lock();
821 let state = modules.entry(module_id.to_string()).or_default();
822 let was_open = state.cooldown_until.is_some();
823 let was_probe = Self::owns_probe(state, probe_token);
824 if was_probe {
825 state.probe_in_flight = None;
826 }
827 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
828 if state.consecutive_timeouts < threshold {
829 return None;
830 }
831 state.cooldown_until = Some(Instant::now() + cooldown);
832 Some(BreakerOpened {
833 consecutive_timeouts: state.consecutive_timeouts,
834 reopened_after_probe: was_open && was_probe,
835 })
836 }
837
838 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
839 state
843 .probe_in_flight
844 .as_ref()
845 .zip(token)
846 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
847 }
848
849 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
850 if let Some(state) = self.lock().get_mut(module_id) {
851 if Self::owns_probe(state, probe_token) {
852 state.probe_in_flight = None;
853 }
854 }
855 }
856
857 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
872 self.lock()
873 .remove(module_id)
874 .map(|state| state.consecutive_timeouts)
875 .filter(|discarded| *discarded > 0)
876 }
877
878 fn open_snapshot(&self) -> Option<serde_json::Value> {
885 let now = Instant::now();
886 let modules = self.lock();
887 let open = modules
888 .iter()
889 .filter_map(|(module_id, state)| {
890 let cooldown_until = state.cooldown_until?;
891 Some((
892 module_id.clone(),
893 serde_json::json!({
894 "consecutive_timeouts": state.consecutive_timeouts,
895 "cooldown_remaining_ms":
896 cooldown_until.saturating_duration_since(now).as_millis() as u64,
897 "probe_in_flight": state.probe_in_flight.is_some(),
898 }),
899 ))
900 })
901 .collect::<serde_json::Map<String, serde_json::Value>>();
902 (!open.is_empty()).then_some(serde_json::Value::Object(open))
903 }
904}
905
906impl ControlHandler {
907 pub fn new(registry: Arc<Registry>) -> Self {
908 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
909 }
910
911 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
912 let counters = forwarding.counters();
913 let route_bind_breakers = forwarding.route_bind_breakers();
917 let route_bind_concurrency = forwarding.route_bind_concurrency();
918 let route_outages = forwarding.route_outages();
919 Self {
920 registry,
921 activity: None,
922 activity_streams: Arc::new(Mutex::new(HashMap::new())),
923 forwarding,
924 process_liveness: None,
925 supervisor: SupervisorHandle::new(),
926 subc_capabilities: Arc::from([
927 CAP_MANIFEST_REGISTRATION.to_string(),
928 CAP_CHANNEL_LIFECYCLE.to_string(),
929 CAP_PING_PONG.to_string(),
930 CAP_SESSION_ATTACH.to_string(),
931 CAP_ADMISSION_FACTS_RELAY.to_string(),
932 CAP_SCOPES_V1.to_string(),
933 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
934 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
935 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
936 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
937 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
938 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
939 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
940 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
941 ]),
942 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
943 route_bind_relay_timeouts: BTreeMap::new(),
944 route_bind_breakers,
945 route_bind_concurrency,
946 route_outages,
947 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
948 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
949 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
950 storage_config: None,
951 machine_id: None,
952 admission_facts_carrier_module_id: None,
953 admission_facts_targets: None,
954 scopes: Arc::new(RwLock::new(ScopeTable::new(
955 crate::daemon_config::default_scope_authority_owners(),
956 ))),
957 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
958 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
959 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
960 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
961 #[cfg(test)]
962 caller_lookup: None,
963 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
964 rescan: None,
965 connected_clients: ConnectedClients::new(),
966 counters,
967 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
968 daemon_provenance: DaemonProvenanceFacts::default(),
969 #[cfg(test)]
970 control_dispatch_delay: None,
971 #[cfg(test)]
972 provenance_probe_override: None,
973 #[cfg(test)]
974 route_bind_relay_sent_hold: None,
975 #[cfg(test)]
976 hello_registry_to_forwarding_hold: None,
977 }
978 }
979
980 pub fn with_storage_config(
983 mut self,
984 storage_config: Option<crate::daemon_config::StorageConfig>,
985 ) -> Self {
986 self.storage_config = storage_config;
987 self
988 }
989
990 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
993 self.activity = Some(crate::activity::ActivityLedger::open(
994 run_dir.join("activity.jsonl"),
995 incarnation,
996 ));
997 self.supervisor.set_activity(self.activity.clone());
998 self
999 }
1000
1001 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1004 self.machine_id = machine_id;
1005 self
1006 }
1007
1008 pub fn with_admission_facts_config(
1012 mut self,
1013 carrier_module_id: Option<String>,
1014 targets: Option<Vec<String>>,
1015 ) -> Self {
1016 self.admission_facts_carrier_module_id = carrier_module_id;
1017 self.admission_facts_targets = targets;
1018 self
1019 }
1020
1021 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1025 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1026 self.scope_authority_owners = owners;
1027 self
1028 }
1029
1030 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1032 self.privileged_operator_prompt = mode;
1033 self
1034 }
1035
1036 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1039 self.route_bind_relay_timeout = timeout;
1040 self
1041 }
1042
1043 pub fn with_route_bind_relay_timeouts(
1049 mut self,
1050 timeouts: impl IntoIterator<Item = (String, Duration)>,
1051 ) -> Self {
1052 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1053 self
1054 }
1055
1056 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1062 self.route_bind_relay_timeouts
1063 .get(module_id)
1064 .copied()
1065 .unwrap_or(self.route_bind_relay_timeout)
1066 }
1067
1068 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1076 self.route_bind_breaker_threshold = threshold.max(1);
1077 self.route_bind_breaker_cooldown = cooldown;
1078 self
1079 }
1080
1081 #[cfg(test)]
1082 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1083 self.health_probe_timeout = timeout;
1084 self
1085 }
1086
1087 #[cfg(test)]
1088 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1089 self.control_dispatch_delay = Some(delay);
1090 self
1091 }
1092
1093 #[cfg(test)]
1097 pub(crate) fn with_route_bind_relay_sent_hold(
1098 mut self,
1099 hold: Arc<tokio::sync::Barrier>,
1100 ) -> Self {
1101 self.route_bind_relay_sent_hold = Some(hold);
1102 self
1103 }
1104
1105 #[cfg(test)]
1110 pub(crate) fn with_hello_registry_to_forwarding_hold(
1111 mut self,
1112 hold: Arc<std::sync::Barrier>,
1113 ) -> Self {
1114 self.hello_registry_to_forwarding_hold = Some(hold);
1115 self
1116 }
1117
1118 #[cfg(test)]
1119 fn hold_between_registry_and_forwarding(&self) {
1120 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1121 hold.wait();
1122 hold.wait();
1123 }
1124 }
1125
1126 pub fn with_process_liveness(
1127 mut self,
1128 process_liveness: Arc<dyn ModuleProcessLiveness>,
1129 ) -> Self {
1130 self.process_liveness = Some(process_liveness);
1131 self
1132 }
1133
1134 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1135 self.supervisor = supervisor;
1136 self.supervisor.set_activity(self.activity.clone());
1137 self
1138 }
1139
1140 pub fn with_daemon_provenance(
1141 mut self,
1142 pid: u32,
1143 started_at_ms: u64,
1144 executable_path: Option<PathBuf>,
1145 build_git_sha: Option<String>,
1146 build_lock_digest: Option<String>,
1147 ) -> Self {
1148 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1149 let process_start_time = process_start_time(pid);
1150 self.daemon_provenance = DaemonProvenanceFacts {
1151 build: DaemonBuildProvenance {
1152 build_git_sha,
1153 build_lock_digest,
1154 },
1155 pid: Some(pid),
1156 started_at_ms: Some(started_at_ms),
1157 start_clock: None,
1158 executable_path,
1159 executable_identity,
1160 process_start_time,
1161 probe: ExecutableIdentityProbe::default(),
1162 };
1163 self
1164 }
1165
1166 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1167 self.daemon_provenance.start_clock = Some(clock);
1168 self
1169 }
1170
1171 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1173 self.scopes
1174 .write()
1175 .unwrap_or_else(|poisoned| poisoned.into_inner())
1176 .set_wall_clock(Arc::new(clock));
1177 self
1178 }
1179
1180 #[cfg(test)]
1181 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1182 self.provenance_probe_override = Some(result);
1183 self
1184 }
1185
1186 pub fn with_capability_config(
1190 self,
1191 modules: impl IntoIterator<Item = (String, bool)>,
1192 reserved_capabilities: BTreeMap<String, String>,
1193 ) -> Self {
1194 self.capability_evaluator
1195 .configure(modules, reserved_capabilities);
1196 self
1197 }
1198
1199 pub fn with_supervisor_rescan(
1200 mut self,
1201 supervisor: Supervisor,
1202 config_path: impl Into<PathBuf>,
1203 configured_port: Option<u16>,
1204 ) -> Self {
1205 self.rescan = Some(SupervisorRescanContext {
1206 supervisor,
1207 config_path: config_path.into(),
1208 configured_port,
1209 storage_config: self.storage_config.clone(),
1210 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1211 admission_facts_targets: self.admission_facts_targets.clone(),
1212 scope_authority_owners: self.scope_authority_owners.clone(),
1213 privileged_operator_prompt: self.privileged_operator_prompt,
1214 });
1215 self
1216 }
1217
1218 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1219 self.connected_clients = connected_clients;
1220 self
1221 }
1222
1223 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1224 Arc::clone(&self.forwarding)
1225 }
1226
1227 pub(crate) fn counters(&self) -> DaemonCounters {
1228 self.counters.clone()
1229 }
1230
1231 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1234 tokio::spawn(async move {
1235 loop {
1236 self.capability_evaluator
1237 .wait_for_change_or_deadline()
1238 .await;
1239 self.refresh_capability_requirements();
1240 }
1241 });
1242 }
1243
1244 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1250 tokio::spawn(async move {
1251 loop {
1252 let has_deadlines = self
1253 .scopes
1254 .read()
1255 .unwrap_or_else(|poisoned| poisoned.into_inner())
1256 .has_deadlines();
1257 if !has_deadlines {
1258 self.scope_deadline_added.notified().await;
1259 continue;
1260 }
1261 tokio::time::sleep(Duration::from_secs(1)).await;
1262 if let Err(error) = self.sweep_expired_scopes() {
1263 warn!(%error, "scope expiry sweep failed");
1264 }
1265 }
1266 });
1267 }
1268
1269 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1270 let mut table = self
1271 .scopes
1272 .write()
1273 .unwrap_or_else(|poisoned| poisoned.into_inner());
1274 let swept = table.sweep_expired();
1275 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1276 drop(table);
1277 self.close_scope_drained_routes(drained);
1278 Ok(())
1279 }
1280
1281 fn publish_scope_changes(
1282 &self,
1283 changes: &[crate::scopes::ScopeTagChange],
1284 expired: &[crate::scopes::ScopeExpired],
1285 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1286 if changes.is_empty() {
1287 return Ok(Vec::new());
1288 }
1289 let drained = self
1290 .forwarding
1291 .publish_scope_changes(changes)
1292 .map_err(RouterError::Forwarding)?;
1293 let mut counts = HashMap::new();
1294 for route in &drained {
1295 *counts
1296 .entry((
1297 route.scope.owner.as_str(),
1298 route.scope.scope_ref.as_str(),
1299 route.scope.tag.scope_epoch,
1300 ))
1301 .or_insert(0usize) += 1;
1302 }
1303 for scope in expired {
1304 let routes_closed = counts
1305 .get(&(
1306 scope.owner.as_str(),
1307 scope.scope_ref.as_str(),
1308 scope.scope_epoch,
1309 ))
1310 .copied()
1311 .unwrap_or(0);
1312 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1313 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1314 routes_closed, "scope expired");
1315 }
1316 Ok(drained)
1317 }
1318
1319 fn runtime_capability_snapshot(
1320 &self,
1321 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1322 let runtime = self
1323 .supervisor
1324 .list()
1325 .into_iter()
1326 .map(|module| {
1327 let status = module.status().map_err(|err| {
1328 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1329 })?;
1330 Ok(RuntimeModule {
1331 module_id: status.module_id,
1332 state: status.state,
1333 enabled: status.enabled,
1334 })
1335 })
1336 .collect::<Result<Vec<_>, RouterError>>()?;
1337 let registrations = self.registry.list_registered_modules().map_err(|err| {
1343 RouterError::backend(
1344 0,
1345 0,
1346 format!("failed to list capability registrations: {err}"),
1347 )
1348 })?;
1349 let registrations = registrations
1350 .into_iter()
1351 .map(|registration| RegisteredModule {
1352 module_id: registration.manifest.module_id,
1353 module_version: registration.manifest.module_version,
1354 capabilities: registration.manifest.capabilities,
1355 })
1356 .collect();
1357 Ok((runtime, registrations))
1358 }
1359
1360 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1367 let cached_registration = RegisteredModule {
1368 module_id: registration.manifest.module_id.clone(),
1369 module_version: registration.manifest.module_version.clone(),
1370 capabilities: registration.manifest.capabilities.clone(),
1371 };
1372 if self.capability_evaluator.record_hello(&cached_registration) {
1373 warn!(
1374 module_id = %cached_registration.module_id,
1375 "capability claims drifted from the cached manifest"
1376 );
1377 }
1378 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1379 self.enforce_capability_denies();
1380 }
1381 self.refresh_capability_requirements();
1382 }
1383
1384 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1388 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1389 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1390 self.supervisor.set_swap_promotion_observer(observer);
1391 }
1392
1393 pub fn refresh_capability_requirements(&self) {
1394 match self.runtime_capability_snapshot() {
1395 Ok((runtime, registrations)) => {
1396 log_requirement_events(
1397 self.capability_evaluator
1398 .evaluate_now(&runtime, ®istrations),
1399 );
1400 }
1401 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1402 }
1403 }
1404
1405 fn enforce_capability_denies(&self) {
1409 let registrations = match self.registry.list_registered_modules() {
1410 Ok(snapshot) => snapshot,
1411 Err(err) => {
1412 warn!(error = %err, "failed to read registrations for capability deny census");
1413 return;
1414 }
1415 };
1416 let manifests = registrations
1417 .into_iter()
1418 .map(|registration| {
1419 (
1420 registration.manifest.module_id.clone(),
1421 registration.manifest,
1422 )
1423 })
1424 .collect::<BTreeMap<_, _>>();
1425 let census = match self.forwarding.route_census(None) {
1426 Ok(census) => census,
1427 Err(err) => {
1428 warn!(error = %err, "failed to read route census for capability deny enforcement");
1429 return;
1430 }
1431 };
1432
1433 for (target_module_id, routes) in census {
1434 let Some(target_manifest) = manifests.get(&target_module_id) else {
1435 continue;
1436 };
1437 let mut closed_routes = Vec::new();
1438 let mut module_goodbyes = Vec::new();
1439 for route in routes {
1440 let Principal::Reserved {
1441 module_id: opening_module_id,
1442 } = &route.principal
1443 else {
1444 continue;
1445 };
1446 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1447 continue;
1448 };
1449 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1450 continue;
1451 };
1452
1453 match self.forwarding.release_client_route(
1454 route.goodbye_target.connection_id,
1455 route.goodbye_target.channel,
1456 route.goodbye_target.epoch,
1457 ) {
1458 Ok(RouteRelease::Removed(module_goodbye)) => {
1459 warn!(
1460 opening_module_id,
1461 target_module_id,
1462 capability,
1463 "force-closing route because an attested capability deny edge now matches"
1464 );
1465 closed_routes.push(route);
1466 module_goodbyes.push(module_goodbye);
1467 }
1468 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1469 Err(err) => warn!(
1470 opening_module_id,
1471 target_module_id,
1472 capability,
1473 error = %err,
1474 "failed to force-close capability-denied route"
1475 ),
1476 }
1477 }
1478
1479 if closed_routes.is_empty() {
1480 continue;
1481 }
1482 send_route_control_pushes(
1483 &self.forwarding,
1484 closed_routes,
1485 ClientControlPush::RouteClosed {
1486 module_id: target_module_id,
1487 channels: Vec::new(),
1488 reason: RouteCloseReason::CapabilityDenied,
1489 drained: false,
1490 abandoned: 0,
1491 excluded_subscriptions: 0,
1492 terminal: Some(false),
1493 },
1494 );
1495 self.emit_route_goodbyes(module_goodbyes);
1496 }
1497 }
1498
1499 fn not_ready_reason(
1505 &self,
1506 registration: &crate::registry::ModuleRegistration,
1507 ) -> Option<NotReadyReason> {
1508 if !registration.ready {
1509 return Some(NotReadyReason {
1510 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1511 capability: None,
1512 });
1513 }
1514 self.first_unprovided_required_capability(registration)
1515 .map(|capability| NotReadyReason {
1516 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1517 capability: Some(capability),
1518 })
1519 }
1520
1521 fn first_unprovided_required_capability(
1539 &self,
1540 registration: &crate::registry::ModuleRegistration,
1541 ) -> Option<String> {
1542 let required = registration
1543 .manifest
1544 .capabilities
1545 .iter()
1546 .flat_map(|declarations| declarations.requires.iter())
1547 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1548 .map(|requirement| requirement.capability.as_str())
1549 .collect::<BTreeSet<_>>();
1550 if required.is_empty() {
1551 return None;
1552 }
1553 let module_id = registration.manifest.module_id.as_str();
1554 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1555 if required
1556 .iter()
1557 .any(|capability| verdict(capability).is_none())
1558 {
1559 self.refresh_capability_requirements();
1560 }
1561 required
1562 .into_iter()
1563 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1564 .map(str::to_string)
1565 }
1566
1567 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1568 self.capability_evaluator
1569 .statuses()
1570 .into_iter()
1571 .map(capability_requirement_status)
1572 .collect()
1573 }
1574
1575 fn deregister_connection(
1582 &self,
1583 connection_id: ConnectionId,
1584 reason: RegistrationEndReason,
1585 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1586 self.registry
1587 .deregister_connection_with_reason(connection_id, reason)
1588 }
1589
1590 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1591 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1592 return None;
1593 }
1594 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1595 parse_client_control_request(&frame.body)
1596 else {
1597 return None;
1598 };
1599 Some(target_module_id(&target).to_string())
1600 }
1601
1602 pub(crate) fn route_open_capacity_refusal(
1603 &self,
1604 ctx: &RouteCtx,
1605 frame: &Frame,
1606 target_module_id: &str,
1607 in_flight: usize,
1608 limit: usize,
1609 ) -> Result<Frame, RouterError> {
1610 self.route_open_admission_refusal_frame(
1611 ctx,
1612 frame,
1613 target_module_id,
1614 "open_admission_full",
1615 (in_flight, limit),
1616 format!(
1617 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1618 ),
1619 )
1620 }
1621
1622 fn route_open_target_capacity_refusal(
1623 &self,
1624 ctx: &RouteCtx,
1625 frame: &Frame,
1626 target_module_id: &str,
1627 in_flight: usize,
1628 ) -> Result<Frame, RouterError> {
1629 self.route_open_admission_refusal_frame(
1630 ctx,
1631 frame,
1632 target_module_id,
1633 "target_binds_full",
1634 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1635 format!(
1636 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1637 ),
1638 )
1639 }
1640
1641 fn route_open_admission_refusal_frame(
1649 &self,
1650 ctx: &RouteCtx,
1651 frame: &Frame,
1652 target_module_id: &str,
1653 reason: &'static str,
1654 (in_flight, limit): (usize, usize),
1655 message: impl Into<String>,
1656 ) -> Result<Frame, RouterError> {
1657 let code = error_codes::TARGET_UNAVAILABLE;
1658 self.counters.increment_route_open_refused(code);
1659 info!(
1660 target: "control",
1661 code,
1662 reason,
1663 module_id = ?target_module_id,
1664 connection_id = ctx.connection_id.get(),
1665 in_flight,
1666 limit,
1667 "route.open refused"
1668 );
1669 control_error_frame(frame, code, message.into())
1670 }
1671
1672 #[cfg(test)]
1678 pub fn handle_control(
1679 &self,
1680 connection_id: ConnectionId,
1681 frame: Frame,
1682 ) -> Result<Vec<Frame>, RouterError> {
1683 match frame.header.ty {
1684 FrameType::Ping => Ok(vec![pong(&frame)?]),
1685 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1686 FrameType::Goodbye => self.handle_goodbye(connection_id),
1687 ty => Ok(vec![control_error_frame(
1688 &frame,
1689 "unsupported_control_frame",
1690 format!("unsupported channel-0 frame {ty:?}"),
1691 )?]),
1692 }
1693 }
1694
1695 pub async fn handle_control_frame(
1696 &self,
1697 ctx: &RouteCtx,
1698 frame: Frame,
1699 ) -> Result<Vec<Frame>, RouterError> {
1700 self.handle_control_frame_timed(ctx, frame, None).await
1701 }
1702
1703 pub(crate) fn record_caller_endpoints(
1704 &self,
1705 connection: ConnectionId,
1706 endpoints: Option<(SocketAddr, SocketAddr)>,
1707 ) {
1708 let mut callers = self
1709 .caller_endpoints
1710 .lock()
1711 .unwrap_or_else(|p| p.into_inner());
1712 callers.remove(&connection);
1713 if let Some(endpoints) = endpoints {
1714 callers.insert(connection, endpoints);
1715 }
1716 }
1717
1718 #[cfg(test)]
1719 pub(crate) fn with_caller_lookup_for_test(
1720 mut self,
1721 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1722 ) -> Self {
1723 self.caller_lookup = Some(Arc::new(lookup));
1724 self
1725 }
1726
1727 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1728 let endpoints = self
1729 .caller_endpoints
1730 .lock()
1731 .unwrap_or_else(|p| p.into_inner())
1732 .get(&connection)
1733 .copied();
1734 #[cfg(test)]
1735 let lookup = self.caller_lookup.clone();
1736 tokio::task::spawn_blocking(move || {
1737 #[cfg(test)]
1738 if let Some(lookup) = lookup {
1739 return lookup(endpoints);
1740 }
1741 endpoints
1742 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1743 .unwrap_or_default()
1744 })
1745 .await
1746 .unwrap_or_default()
1747 }
1748
1749 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1750 frame.header.channel == 0
1751 && frame.header.ty == FrameType::Request
1752 && self
1753 .forwarding
1754 .module_endpoint_for_connection(ctx.connection_id)
1755 .is_ok_and(|endpoint| endpoint.is_none())
1756 && parse_client_control_request(&frame.body)
1757 .ok()
1758 .is_some_and(|request| privileged_operation(&request).is_some())
1759 }
1760
1761 pub(crate) async fn handle_control_frame_timed(
1762 &self,
1763 ctx: &RouteCtx,
1764 frame: Frame,
1765 dispatch_started_at: Option<StdInstant>,
1766 ) -> Result<Vec<Frame>, RouterError> {
1767 match frame.header.ty {
1768 FrameType::Ping => Ok(vec![pong(&frame)?]),
1769 FrameType::Hello => {
1770 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1771 }
1772 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1773 FrameType::Cancel => {
1774 if self
1777 .forwarding
1778 .operator_confirms()
1779 .cancel(ctx.connection_id, frame.header.corr)
1780 || self
1781 .supervisor
1782 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1783 || self.cancel_activity_subscription(ctx, &frame)?
1784 {
1785 Ok(Vec::new())
1786 } else {
1787 Ok(vec![control_error_frame(
1788 &frame,
1789 "unknown_subscription",
1790 "no subscription has this correlation id",
1791 )?])
1792 }
1793 }
1794 FrameType::Request => {
1795 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1798 if op
1799 .as_ref()
1800 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1801 {
1802 return self.handle_activity_record(ctx, frame).await;
1803 }
1804 if op.as_ref().is_some_and(|probe| {
1805 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1806 }) {
1807 return self.handle_activity_read(ctx, frame).await;
1808 }
1809 if op
1810 .as_ref()
1811 .is_some_and(|probe| probe.op == "operator.confirm")
1812 {
1813 return self.handle_operator_confirm(ctx, frame);
1814 }
1815 if op
1816 .as_ref()
1817 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1818 {
1819 return self.handle_scope_served_by(ctx.connection_id, frame);
1820 }
1821 if self
1822 .forwarding
1823 .module_endpoint_for_connection(ctx.connection_id)
1824 .map_err(RouterError::Forwarding)?
1825 .is_some()
1826 {
1827 if !is_known_module_request_op(&frame.body) {
1828 return Ok(vec![control_error_frame(
1829 &frame,
1830 "unsupported_control_frame",
1831 "module-originated channel-0 REQUEST is not supported",
1832 )?]);
1833 }
1834 let request = match parse_module_control_request_from_module(&frame.body) {
1835 Ok(request) => request,
1836 Err((err, ControlRequestBodyError::UnknownOp)) => {
1837 return Ok(vec![control_error_frame(
1838 &frame,
1839 "unsupported_control_frame",
1840 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1841 )?])
1842 }
1843 Err((err, ControlRequestBodyError::InvalidBody)) => {
1844 return Ok(vec![control_error_frame(
1845 &frame,
1846 "invalid_control_body",
1847 format!("malformed module control body: {err}"),
1848 )?])
1849 }
1850 };
1851 let op = module_control_request_op(&request);
1852 let corr = frame.header.corr;
1853 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1854 let result =
1855 self.handle_module_control_request(ctx.connection_id, frame, request);
1856 log_slow_control_dispatch(
1857 dispatch_started_at,
1858 op,
1859 ctx.connection_id,
1860 corr,
1861 None,
1862 );
1863 return result;
1864 }
1865
1866 if is_known_module_request_op(&frame.body) {
1867 return Ok(vec![control_error_frame(
1868 &frame,
1869 "not_registered",
1870 "catalog.update requires an active module registration owned by this connection",
1871 )?]);
1872 }
1873
1874 let request = match parse_client_control_request(&frame.body) {
1875 Ok(request) => request,
1876 Err((err, ControlRequestBodyError::UnknownOp)) => {
1877 return Ok(vec![control_error_frame(
1878 &frame,
1879 "unknown_control_op",
1880 format!("unknown client control op: {err}"),
1881 )?])
1882 }
1883 Err((err, ControlRequestBodyError::InvalidBody)) => {
1884 return Ok(vec![control_error_frame(
1885 &frame,
1886 "invalid_control_body",
1887 format!("malformed client control body: {err}"),
1888 )?])
1889 }
1890 };
1891 let op = client_control_request_op(&request);
1892 let corr = frame.header.corr;
1893 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1894 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1895 #[cfg(test)]
1896 if let Some(delay) = self.control_dispatch_delay {
1897 tokio::time::sleep(delay).await;
1898 }
1899 let result = self
1900 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1901 .await;
1902 if let Some(timings) = route_open_timings.as_mut() {
1903 timings.finish();
1904 }
1905 log_slow_control_dispatch(
1906 dispatch_started_at,
1907 op,
1908 ctx.connection_id,
1909 corr,
1910 route_open_timings.as_ref(),
1911 );
1912 result
1913 }
1914 FrameType::Push => {
1915 let Some(endpoint) = self
1916 .forwarding
1917 .module_endpoint_for_connection(ctx.connection_id)
1918 .map_err(RouterError::Forwarding)?
1919 else {
1920 return Ok(vec![control_error_frame(
1921 &frame,
1922 "unsupported_control_frame",
1923 "client-originated channel-0 PUSH is not supported",
1924 )?]);
1925 };
1926 self.handle_status_update(endpoint, frame)
1927 }
1928 FrameType::Response | FrameType::Error
1929 if self
1930 .forwarding
1931 .module_endpoint_for_connection(ctx.connection_id)
1932 .map_err(RouterError::Forwarding)?
1933 .is_some() =>
1934 {
1935 self.handle_module_relay_response(ctx.connection_id, frame)
1936 }
1937 ty => Ok(vec![control_error_frame(
1938 &frame,
1939 "unsupported_control_frame",
1940 format!("unsupported channel-0 frame {ty:?}"),
1941 )?]),
1942 }
1943 }
1944
1945 pub fn cleanup_connection(
1946 &self,
1947 connection_id: ConnectionId,
1948 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1949 self.caller_endpoints
1950 .lock()
1951 .unwrap_or_else(|p| p.into_inner())
1952 .remove(&connection_id);
1953 self.forwarding
1954 .operator_confirms()
1955 .connection_closed(connection_id);
1956 self.cleanup_connection_with_end_reason(
1957 connection_id,
1958 RegistrationEndReason::ConnectionClosed,
1959 )
1960 }
1961
1962 fn cleanup_connection_with_end_reason(
1963 &self,
1964 connection_id: ConnectionId,
1965 requested_reason: RegistrationEndReason,
1966 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1967 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1968 self.registry
1969 .get_module_by_connection(connection_id)?
1970 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1971 .and_then(|module| module.registration_end_reason().ok().flatten())
1972 .unwrap_or(requested_reason)
1973 } else {
1974 requested_reason
1975 };
1976 let crash_closed = self
1977 .registry
1978 .get_module_by_connection(connection_id)?
1979 .and_then(|registration| {
1980 self.forwarding
1981 .module_endpoint_for_connection(connection_id)
1982 .ok()
1983 .flatten()
1984 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1985 .map(|routes| (registration.manifest.module_id, routes))
1986 });
1987 let crash_closed = crash_closed.map(|(module_id, routes)| {
1988 let terminal = match self.supervisor.get(&module_id) {
1989 None => false,
1990 Some(module) => match module.will_recover_after_connection_loss() {
1991 Ok(will_recover) => !will_recover,
1992 Err(err) => {
1993 warn!(
1994 %module_id,
1995 error = %err,
1996 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1997 );
1998 false
1999 }
2000 },
2001 };
2002 let (reason, drained) = match self
2006 .forwarding
2007 .daemon_drain_result_for_connection(connection_id)
2008 {
2009 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2010 Ok(None) => (RouteCloseReason::Crash, false),
2011 Err(err) => {
2012 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2013 (RouteCloseReason::Crash, false)
2014 }
2015 };
2016 (module_id, routes, reason, drained, terminal)
2017 });
2018 let registrations = self.deregister_connection(connection_id, end_reason);
2019 let cleanup = if crash_closed.is_some() {
2020 self.forwarding.cleanup_connection_counted(connection_id)
2021 } else {
2022 self.forwarding
2028 .cleanup_connection(connection_id)
2029 .map(|released| crate::forwarding::ConnectionCleanup {
2030 released,
2031 abandoned_relays: 0,
2032 })
2033 };
2034 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2039 let abandoned = cleanup
2040 .as_ref()
2041 .map_or(0, |cleanup| cleanup.abandoned_relays);
2042 send_route_control_pushes(
2043 &self.forwarding,
2044 routes,
2045 ClientControlPush::RouteClosed {
2046 module_id,
2047 channels: Vec::new(),
2048 reason,
2049 drained,
2050 abandoned,
2051 excluded_subscriptions: 0,
2052 terminal: Some(terminal),
2053 },
2054 );
2055 }
2056 if let Ok(cleanup) = cleanup {
2057 self.emit_route_goodbyes(cleanup.released);
2058 }
2059 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2063 crate::supervise::notify_registration_release();
2064 self.capability_evaluator.wake_deadline_loop();
2065 self.refresh_capability_requirements();
2066 }
2067 self.supervisor.remove_spawn_subscribers(connection_id);
2068 self.activity_streams
2069 .lock()
2070 .unwrap_or_else(|p| p.into_inner())
2071 .retain(|(id, _), task| {
2072 if *id == connection_id {
2073 task.abort();
2074 false
2075 } else {
2076 true
2077 }
2078 });
2079 self.hello_launch_nonces
2082 .lock()
2083 .unwrap_or_else(|poisoned| poisoned.into_inner())
2084 .forget(connection_id);
2085 self.scopes
2086 .write()
2087 .unwrap_or_else(|poisoned| poisoned.into_inner())
2088 .release_connection(connection_id);
2089 registrations
2090 }
2091
2092 pub(crate) fn handle_route_goodbye(
2093 &self,
2094 connection_id: ConnectionId,
2095 route_channel: u16,
2096 route_epoch: u32,
2097 ) -> Result<bool, RouterError> {
2098 debug!(
2099 connection_id = connection_id.get(),
2100 route_channel, route_epoch, "handling route GOODBYE"
2101 );
2102 let RouteRelease::Removed(released_route) = self
2103 .forwarding
2104 .release_client_route(connection_id, route_channel, route_epoch)
2105 .map_err(RouterError::Forwarding)?
2106 else {
2107 return Ok(false);
2108 };
2109 self.emit_route_goodbyes(vec![released_route]);
2110 Ok(true)
2111 }
2112
2113 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2114 for released in released_routes {
2115 let frame = match Frame::build_with_version(
2116 released.negotiated_ver,
2117 FrameType::Goodbye,
2118 control_flags(),
2119 released.channel,
2120 released.epoch,
2121 0,
2122 Vec::new(),
2123 ) {
2124 Ok(frame) => frame,
2125 Err(err) => {
2126 warn!(
2127 route_channel = released.channel,
2128 error = %err,
2129 "failed to build route GOODBYE frame"
2130 );
2131 continue;
2132 }
2133 };
2134 if !released.close_on_delivery_failure() {
2135 crate::forwarding::send_module_route_goodbye(
2136 &self.counters,
2137 &released.sink,
2138 frame,
2139 released.module_id.as_deref(),
2140 "client route released",
2141 );
2142 continue;
2143 }
2144 if let Err(err) = released.sink.try_send(frame) {
2145 warn!(
2146 target_connection_id = released.connection_id.get(),
2147 route_channel = released.channel,
2148 error = %err,
2149 "route GOODBYE was not delivered to client; closing target connection"
2150 );
2151 if self
2152 .forwarding
2153 .escalate_client_delivery_failure(
2154 released.connection_id,
2155 released.channel,
2156 released.epoch,
2157 CloseReason::new(
2158 "route_goodbye_delivery_failed",
2159 format!(
2160 "failed to enqueue route GOODBYE for channel {}: {err}",
2161 released.channel
2162 ),
2163 ),
2164 crate::forwarding::UndeliveredFrame {
2165 module_id: released.module_id.as_deref(),
2166 sink: &released.sink,
2167 },
2168 )
2169 .unwrap_or(false)
2170 {
2171 self.counters.increment_goodbye_relay_client_failed();
2172 }
2173 }
2174 }
2175 }
2176
2177 fn send_abandoned_route_bind_goodbye(
2189 &self,
2190 module_sink: &crate::FrameSink,
2191 negotiated_ver: u8,
2192 module_channel: u16,
2193 module_epoch: u32,
2194 ) {
2195 let frame = match Frame::build_with_version(
2196 negotiated_ver,
2197 FrameType::Goodbye,
2198 control_flags(),
2199 module_channel,
2200 module_epoch,
2201 0,
2202 Vec::new(),
2203 ) {
2204 Ok(frame) => frame,
2205 Err(err) => {
2206 warn!(
2207 route_channel = module_channel,
2208 error = %err,
2209 "failed to build GOODBYE for abandoned route.bind"
2210 );
2211 return;
2212 }
2213 };
2214 crate::forwarding::send_module_route_goodbye(
2215 &self.counters,
2216 module_sink,
2217 frame,
2218 None,
2219 "abandoned route.bind",
2220 );
2221 }
2222
2223 fn handle_hello(
2224 &self,
2225 connection_id: ConnectionId,
2226 sink: Option<crate::FrameSink>,
2227 frame: Frame,
2228 ) -> Result<Vec<Frame>, RouterError> {
2229 debug!(
2230 connection_id = connection_id.get(),
2231 corr = frame.header.corr,
2232 "handling HELLO"
2233 );
2234 if self
2238 .registry
2239 .get_module_by_connection(connection_id)
2240 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2241 .is_some()
2242 {
2243 return Ok(vec![control_error_frame(
2244 &frame,
2245 "invalid_hello",
2246 "connection is already registered as a module",
2247 )?]);
2248 }
2249 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2250 Ok(value) => value,
2251 Err(err) => {
2252 return Ok(vec![control_error_frame(
2253 &frame,
2254 "invalid_hello",
2255 format!("malformed HELLO body: {err}"),
2256 )?])
2257 }
2258 };
2259 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2260 return Ok(vec![control_error_frame(
2261 &frame,
2262 "invalid_capability_grammar",
2263 err.to_string(),
2264 )?]);
2265 }
2266 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2267 return Ok(vec![control_error_frame(
2268 &frame,
2269 "invalid_manifest",
2270 err.to_string(),
2271 )?]);
2272 }
2273 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2274 return Ok(vec![control_error_frame(
2275 &frame,
2276 "invalid_event_declaration",
2277 err.to_string(),
2278 )?]);
2279 }
2280 if let Some(provenance) = hello_value
2281 .get("manifest")
2282 .and_then(|manifest| manifest.get("provenance"))
2283 {
2284 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2285 return Ok(vec![control_error_frame(
2286 &frame,
2287 "invalid_manifest",
2288 format!("malformed manifest provenance: {err}"),
2289 )?]);
2290 }
2291 }
2292 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2293 Ok(hello) => hello,
2294 Err(err) => {
2295 return Ok(vec![control_error_frame(
2296 &frame,
2297 "invalid_hello",
2298 format!("malformed HELLO body: {err}"),
2299 )?])
2300 }
2301 };
2302
2303 if hello.protocol_ver != hello.manifest.protocol_ver {
2304 return Ok(vec![control_error_frame(
2305 &frame,
2306 "invalid_manifest",
2307 format!(
2308 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2309 hello.protocol_ver, hello.manifest.protocol_ver
2310 ),
2311 )?]);
2312 }
2313
2314 if hello.manifest.module_id.trim().is_empty() {
2315 return Ok(vec![control_error_frame(
2316 &frame,
2317 "invalid_manifest",
2318 "manifest module_id must not be empty",
2319 )?]);
2320 }
2321
2322 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2323 Ok(negotiated_ver) => negotiated_ver,
2324 Err(message) => {
2325 return Ok(vec![control_error_frame(
2326 &frame,
2327 "version_unsupported",
2328 message,
2329 )?])
2330 }
2331 };
2332
2333 let swap_admission = self
2341 .supervisor
2342 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2343 if swap_admission == SwapHelloAdmission::Refused {
2344 warn!(
2345 module_id = %hello.manifest.module_id,
2346 connection_id = connection_id.get(),
2347 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2348 );
2349 return Ok(vec![control_error_frame(
2350 &frame,
2351 "swap_token_invalid",
2352 format!(
2353 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2354 hello.manifest.module_id
2355 ),
2356 )?]);
2357 }
2358 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2359
2360 if let Some(rejection) = (!swap_candidate)
2368 .then(|| {
2369 self.supervisor.reserved_hello_rejection(
2370 &hello.manifest.module_id,
2371 hello.launch_nonce.as_deref(),
2372 )
2373 })
2374 .flatten()
2375 {
2376 let message = match rejection {
2377 ReservedHelloRejection::Exact { module_id } => format!(
2378 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2379 ),
2380 ReservedHelloRejection::Prefix {
2381 prefix,
2382 owner_module_id,
2383 } => format!(
2384 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2385 hello.manifest.module_id
2386 ),
2387 };
2388 return Ok(vec![control_error_frame(
2389 &frame,
2390 "reserved_module",
2391 message,
2392 )?]);
2393 }
2394
2395 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2396 &hello.manifest.module_id,
2397 hello.manifest.capabilities.as_ref(),
2398 );
2399 if let Some(refusal) = reserved_capability_refusals.first() {
2400 let capability = refusal.capability.clone();
2401 let bound_module = refusal.claimants[0].clone();
2402 log_duplicate_claim_events(reserved_capability_refusals);
2403 return Ok(vec![control_error_frame(
2404 &frame,
2405 "reserved_capability",
2406 format!(
2407 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2408 capability, bound_module, hello.manifest.module_id
2409 ),
2410 )?]);
2411 }
2412
2413 if self
2416 .forwarding
2417 .connection_has_client_routes(connection_id)
2418 .map_err(RouterError::Forwarding)?
2419 {
2420 return Ok(vec![control_error_frame(
2421 &frame,
2422 "invalid_hello",
2423 "connection has open client routes and cannot also register as a module",
2424 )?]);
2425 }
2426
2427 self.hello_launch_nonces
2432 .lock()
2433 .unwrap_or_else(|poisoned| poisoned.into_inner())
2434 .record(connection_id, hello.launch_nonce.as_deref());
2435 let control_ops = effective_module_control_ops(hello.control_ops);
2436 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2439 if swap_candidate {
2440 return self.register_swap_candidate(
2441 connection_id,
2442 sink,
2443 &frame,
2444 hello.manifest,
2445 negotiated_ver,
2446 control_ops,
2447 hello_ack,
2448 );
2449 }
2450 let registration = match self.registry.register_unpublished_with_control_ops(
2457 hello.manifest,
2458 negotiated_ver,
2459 connection_id,
2460 control_ops,
2461 ) {
2462 Ok(registration) => registration,
2463 Err(RegistryError::DuplicateModuleId { module_id }) => {
2464 return Ok(vec![control_error_frame(
2465 &frame,
2466 "duplicate_module_id",
2467 format!(
2468 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2469 ),
2470 )?])
2471 }
2472 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2473 return Ok(vec![control_error_frame(
2474 &frame,
2475 "invalid_module_id",
2476 err.to_string(),
2477 )?])
2478 }
2479 Err(err) => {
2480 return Ok(vec![control_error_frame(
2481 &frame,
2482 "registry_error",
2483 err.to_string(),
2484 )?])
2485 }
2486 };
2487 #[cfg(test)]
2488 self.hold_between_registry_and_forwarding();
2489
2490 let reply = if let Some(sink) = sink {
2491 let concurrency = manifest_concurrency(®istration.manifest);
2507 if let Err(err) = self.forwarding.register_module_connection_acked(
2508 connection_id,
2509 registration.manifest.module_id.clone(),
2510 negotiated_ver,
2511 concurrency,
2512 sink,
2513 hello_ack,
2514 ) {
2515 if matches!(
2520 self.deregister_connection(
2521 connection_id,
2522 RegistrationEndReason::RegistrationFailed,
2523 ),
2524 Ok(r) if !r.is_empty()
2525 ) {
2526 crate::supervise::notify_registration_release();
2527 }
2528 return Ok(vec![control_error_frame(
2529 &frame,
2530 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2531 "invalid_hello"
2532 } else {
2533 forwarding_error_code(&err)
2534 },
2535 err.to_string(),
2536 )?]);
2537 }
2538 Vec::new()
2539 } else {
2540 vec![hello_ack]
2543 };
2544
2545 if !self
2550 .registry
2551 .publish_registration(connection_id)
2552 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2553 {
2554 debug!(
2555 module_id = %registration.manifest.module_id,
2556 connection_id = connection_id.get(),
2557 "registration was removed before it could be published"
2558 );
2559 }
2560
2561 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2570 info!(
2571 module_id = %registration.manifest.module_id,
2572 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2573 );
2574 }
2575
2576 self.apply_registration_capabilities(®istration);
2577
2578 self.supervisor
2579 .record_activity_registration(®istration.manifest.module_id);
2580
2581 info!(
2582 module_id = %registration.manifest.module_id,
2583 module_version = %registration.manifest.module_version,
2584 negotiated_ver,
2585 routable_provider = manifest_provides_routable_role(®istration.manifest),
2586 connection_id = connection_id.get(),
2587 "module registered"
2588 );
2589
2590 Ok(reply)
2591 }
2592
2593 #[allow(clippy::too_many_arguments)]
2610 fn register_swap_candidate(
2611 &self,
2612 connection_id: ConnectionId,
2613 sink: Option<crate::FrameSink>,
2614 frame: &Frame,
2615 manifest: ModuleManifest,
2616 negotiated_ver: u8,
2617 control_ops: Vec<String>,
2618 hello_ack: Frame,
2619 ) -> Result<Vec<Frame>, RouterError> {
2620 let module_id = manifest.module_id.clone();
2621 let registration = match self.registry.register_unpublished_candidate_with_control_ops(
2622 manifest,
2623 negotiated_ver,
2624 connection_id,
2625 control_ops,
2626 ) {
2627 Ok(registration) => registration,
2628 Err(RegistryError::DuplicateModuleId { module_id }) => {
2629 return Ok(vec![control_error_frame(
2630 frame,
2631 "duplicate_module_id",
2632 format!(
2633 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2634 ),
2635 )?])
2636 }
2637 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2638 return Ok(vec![control_error_frame(
2639 frame,
2640 "invalid_module_id",
2641 err.to_string(),
2642 )?])
2643 }
2644 Err(err) => {
2645 return Ok(vec![control_error_frame(
2646 frame,
2647 "registry_error",
2648 err.to_string(),
2649 )?])
2650 }
2651 };
2652 #[cfg(test)]
2653 self.hold_between_registry_and_forwarding();
2654 let reply = if let Some(sink) = sink {
2655 let concurrency = manifest_concurrency(®istration.manifest);
2659 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2660 connection_id,
2661 module_id.clone(),
2662 negotiated_ver,
2663 concurrency,
2664 sink,
2665 hello_ack,
2666 ) {
2667 if matches!(
2668 self.deregister_connection(
2669 connection_id,
2670 RegistrationEndReason::RegistrationFailed,
2671 ),
2672 Ok(r) if !r.is_empty()
2673 ) {
2674 crate::supervise::notify_registration_release();
2675 }
2676 return Ok(vec![control_error_frame(
2677 frame,
2678 forwarding_error_code(&err),
2679 err.to_string(),
2680 )?]);
2681 }
2682 Vec::new()
2683 } else {
2684 vec![hello_ack]
2685 };
2686 if !self
2687 .registry
2688 .publish_registration(connection_id)
2689 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2690 {
2691 debug!(
2692 module_id = %module_id,
2693 connection_id = connection_id.get(),
2694 "swap candidate registration was removed before it could be published"
2695 );
2696 }
2697 self.supervisor.mark_swap_candidate_admitted(&module_id);
2698 info!(
2699 module_id = %module_id,
2700 module_version = %registration.manifest.module_version,
2701 negotiated_ver,
2702 ready = registration.ready,
2703 connection_id = connection_id.get(),
2704 "swap candidate registered; not routable until cutover"
2705 );
2706 Ok(reply)
2707 }
2708
2709 fn build_hello_ack(
2710 &self,
2711 frame: &Frame,
2712 negotiated_ver: u8,
2713 module_id: &str,
2714 ) -> Result<Frame, RouterError> {
2715 let ack = ModuleHelloAckBody {
2716 negotiated_ver,
2717 subc_ops: module_subc_ops(),
2718 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2719 storage: self
2720 .storage_config
2721 .as_ref()
2722 .map(|cfg| cfg.descriptor_for(module_id)),
2723 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2724 };
2725 let body = serde_json::to_vec(&ack).map_err(|err| {
2726 RouterError::backend(
2727 0,
2728 frame.header.corr,
2729 format!("failed to encode HELLO_ACK: {err}"),
2730 )
2731 })?;
2732
2733 Frame::build_with_version(
2734 negotiated_ver,
2735 FrameType::HelloAck,
2736 control_flags(),
2737 0,
2738 0,
2739 frame.header.corr,
2740 body,
2741 )
2742 .map_err(RouterError::FrameBuild)
2743 }
2744
2745 async fn handle_client_control_request(
2746 &self,
2747 ctx: &RouteCtx,
2748 frame: Frame,
2749 request: ClientControlRequest,
2750 route_open_timings: &mut Option<RouteOpenTimings>,
2751 ) -> Result<Vec<Frame>, RouterError> {
2752 if let Some(operation) = privileged_operation(&request) {
2753 let caller = self.resolve_caller(ctx.connection_id).await;
2754 let op = client_control_request_op(&request);
2755 let text = prompt_text(operation, &caller);
2756 match self.privileged_operator_prompt {
2757 PrivilegedOperatorPrompt::Log => {
2758 info!(target: "subc_daemon::privileged_prompt", operation = op,
2759 target = ?operation, caller = %caller.description(), summary = %text,
2760 "privileged request would prompt the operator");
2761 }
2762 PrivilegedOperatorPrompt::Enforce => {
2763 let key =
2764 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2765 let outcome = self
2766 .forwarding
2767 .operator_confirms()
2768 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2769 .await;
2770 if outcome.code != "confirmed" {
2771 return Ok(vec![outcome.frame(&frame)]);
2772 }
2773 }
2774 }
2775 }
2776 match request {
2777 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2778 ClientControlRequest::CatalogList { module_id } => {
2779 self.handle_catalog_list(frame, module_id)
2780 }
2781 ClientControlRequest::RouteOpen {
2782 target,
2783 identity,
2784 consumer_identity,
2785 consumer_capabilities,
2786 role_versions,
2787 admission_facts,
2788 scope,
2789 } => {
2790 self.handle_route_open(
2791 ctx,
2792 frame,
2793 RouteOpenRequest {
2794 target,
2795 identity,
2796 consumer_identity,
2797 consumer_capabilities,
2798 role_versions,
2799 admission_facts,
2800 scope,
2801 },
2802 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2803 )
2804 .await
2805 }
2806 ClientControlRequest::RoutePoll {
2807 route_channel,
2808 route_epoch,
2809 kind,
2810 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2811 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2812 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2813 self.handle_supervisor_spawn_snapshot(frame)
2814 }
2815 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2816 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2817 }
2818 ClientControlRequest::SupervisorRestart {
2819 module_id,
2820 drain_timeout_ms,
2821 } => {
2822 self.log_supervisor_request_received(
2823 ctx,
2824 frame.header.corr,
2825 ops::SUPERVISOR_RESTART,
2826 Some(&module_id),
2827 None,
2828 )?;
2829 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2830 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2831 .await
2832 }
2833 ClientControlRequest::SupervisorSwap {
2834 module_id,
2835 ready_timeout_ms,
2836 } => {
2837 self.log_supervisor_request_received(
2838 ctx,
2839 frame.header.corr,
2840 ops::SUPERVISOR_SWAP,
2841 Some(&module_id),
2842 None,
2843 )?;
2844 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2845 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2846 .await
2847 }
2848 ClientControlRequest::SupervisorReload { module_id } => {
2849 self.log_supervisor_request_received(
2850 ctx,
2851 frame.header.corr,
2852 ops::SUPERVISOR_RELOAD,
2853 Some(&module_id),
2854 None,
2855 )?;
2856 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2857 self.handle_supervisor_reload(frame, module_id, requester)
2858 .await
2859 }
2860 ClientControlRequest::SupervisorRescan { preview } => {
2861 if !preview {
2862 self.log_supervisor_request_received(
2863 ctx,
2864 frame.header.corr,
2865 ops::SUPERVISOR_RESCAN,
2866 None,
2867 None,
2868 )?;
2869 }
2870 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2871 self.handle_supervisor_rescan(frame, preview, requester)
2872 .await
2873 }
2874 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2875 self.log_supervisor_request_received(
2876 ctx,
2877 frame.header.corr,
2878 ops::SUPERVISOR_RELEASE_RESERVED,
2879 Some(&module_id),
2880 None,
2881 )?;
2882 self.handle_supervisor_release_reserved(frame, module_id)
2883 .await
2884 }
2885 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2886 self.log_supervisor_request_received(
2887 ctx,
2888 frame.header.corr,
2889 ops::SUPERVISOR_SET_ENABLED,
2890 Some(&module_id),
2891 Some(enabled),
2892 )?;
2893 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2894 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2895 .await
2896 }
2897 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2898 self.handle_supervisor_health_probe(frame, module_id).await
2899 }
2900 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2901 ClientControlRequest::SupervisorRoutes { module_id } => {
2902 self.handle_supervisor_routes(frame, module_id)
2903 }
2904 ClientControlRequest::SupervisorProvenance { module_id } => {
2905 self.handle_supervisor_provenance(frame, module_id).await
2906 }
2907 ClientControlRequest::SupervisorStderrTail {
2908 module_id,
2909 max_lines,
2910 max_bytes,
2911 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2912 ClientControlRequest::SupervisorTerminals { module_id } => {
2913 self.handle_supervisor_terminals(frame, module_id).await
2914 }
2915 }
2916 }
2917
2918 fn log_supervisor_request_received(
2919 &self,
2920 ctx: &RouteCtx,
2921 corr: u64,
2922 op: &'static str,
2923 module_id: Option<&str>,
2924 enabled: Option<bool>,
2925 ) -> Result<(), RouterError> {
2926 let caller = self.supervisor_requester(ctx, corr)?;
2927 let caller = principal_label(&caller);
2928 info!(
2931 target: "control",
2932 op,
2933 module_id,
2934 enabled,
2935 connection_id = ctx.connection_id.get(),
2936 caller = %caller,
2937 "supervisor request received"
2938 );
2939 Ok(())
2940 }
2941
2942 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2943 Ok(self
2944 .registry
2945 .get_module_by_connection(ctx.connection_id)
2946 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2947 .map(|registration| Principal::Reserved {
2948 module_id: registration.manifest.module_id,
2949 })
2950 .unwrap_or(Principal::Direct))
2951 }
2952
2953 fn handle_module_control_request(
2954 &self,
2955 connection_id: ConnectionId,
2956 frame: Frame,
2957 request: ModuleControlRequestFromModule,
2958 ) -> Result<Vec<Frame>, RouterError> {
2959 match request {
2960 ModuleControlRequestFromModule::CatalogUpdate {
2961 provides,
2962 capabilities,
2963 ready,
2964 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2965 ModuleControlRequestFromModule::LiveRoots {} => {
2966 let registered = self
2967 .registry
2968 .get_module_by_connection(connection_id)
2969 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2970 let Some(registration) = registered else {
2971 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2972 };
2973 let response = self
2974 .forwarding
2975 .live_roots(®istration.manifest.module_id)
2976 .map_err(RouterError::Forwarding)?;
2977 Ok(vec![control_response_body_frame(
2978 &frame,
2979 &response,
2980 "ModuleControlResponseToModule::LiveRoots",
2981 )?])
2982 }
2983 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2984 self.handle_scope_sync(connection_id, frame, generation, scopes)
2985 }
2986 ModuleControlRequestFromModule::ScopeApply {
2987 generation,
2988 upsert,
2989 end,
2990 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2991 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2992 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2993 }
2994 }
2995 }
2996
2997 fn handle_scope_served_by(
2998 &self,
2999 connection_id: ConnectionId,
3000 frame: Frame,
3001 ) -> Result<Vec<Frame>, RouterError> {
3002 let registered = self
3003 .registry
3004 .get_module_by_connection(connection_id)
3005 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3006 let Some(registered) = registered else {
3007 return Ok(vec![control_error_frame(
3008 &frame,
3009 "not_registered",
3010 "scope.served_by requires an active module registration owned by this connection",
3011 )?]);
3012 };
3013 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3014 {
3015 Ok(request) => request,
3016 Err(err) => {
3017 return Ok(vec![control_error_frame(
3018 &frame,
3019 "invalid_control_body",
3020 format!("malformed scope.served_by body: {err}"),
3021 )?])
3022 }
3023 };
3024 let answer = self
3025 .scopes
3026 .read()
3027 .unwrap_or_else(|poisoned| poisoned.into_inner())
3028 .served_by(
3029 ®istered.manifest.module_id,
3030 &request.scope_ref,
3031 request.scope_epoch,
3032 &request.module_id,
3033 );
3034 let reply = subc_control::ScopeServedByReply::new(
3035 answer,
3036 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3037 );
3038 Ok(vec![control_response_body_frame(
3039 &frame,
3040 &reply,
3041 "ScopeServedByReply",
3042 )?])
3043 }
3044
3045 fn handle_operator_confirm(
3046 &self,
3047 ctx: &RouteCtx,
3048 frame: Frame,
3049 ) -> Result<Vec<Frame>, RouterError> {
3050 use crate::operator_confirm::{audit, Outcome};
3051 let registration = self
3052 .registry
3053 .get_module_by_connection(ctx.connection_id)
3054 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3055 let Some(registration) = registration else {
3056 let outcome = Outcome::refusal("not_registered");
3057 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3058 return Ok(vec![outcome.frame(&frame)]);
3059 };
3060 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3061 Ok(request) => request,
3062 Err(_) => {
3063 let outcome = Outcome::refusal("invalid_control_body");
3064 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3065 return Ok(vec![outcome.frame(&frame)]);
3066 }
3067 };
3068 let module_id = registration.manifest.module_id;
3069 let nonce = self
3073 .hello_launch_nonces
3074 .lock()
3075 .unwrap_or_else(|p| p.into_inner())
3076 .nonce(ctx.connection_id)
3077 .map(str::to_owned);
3078 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3079 self.supervisor
3080 .spawned_consumer_authorized(&module_id, nonce)
3081 });
3082 let confirms = self.forwarding.operator_confirms();
3083 self.forwarding
3084 .with_operator_route(
3085 ctx.connection_id,
3086 request.route_channel,
3087 request.route_epoch,
3088 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3089 )
3090 .map_err(RouterError::Forwarding)
3091 }
3092
3093 fn handle_scope_sync(
3097 &self,
3098 connection_id: ConnectionId,
3099 frame: Frame,
3100 generation: u64,
3101 scopes: Vec<ScopeRecord>,
3102 ) -> Result<Vec<Frame>, RouterError> {
3103 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3104 }
3105
3106 fn handle_scope_change(
3107 &self,
3108 connection_id: ConnectionId,
3109 frame: Frame,
3110 generation: u64,
3111 scopes: Vec<ScopeRecord>,
3112 end: Option<Vec<ScopeEnd>>,
3113 ) -> Result<Vec<Frame>, RouterError> {
3114 let is_apply = end.is_some();
3115 let Some(registration) = self
3116 .registry
3117 .get_module_by_connection(connection_id)
3118 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3119 else {
3120 return Ok(vec![control_error_frame(
3121 &frame,
3122 "not_registered",
3123 "scope.sync and scope.apply require an active module registration owned by this connection",
3124 )?]);
3125 };
3126 let owner = registration.manifest.module_id;
3127 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3128 let is_current_launch = |connection: ConnectionId| {
3129 self.hello_launch_nonces
3130 .lock()
3131 .unwrap_or_else(|poisoned| poisoned.into_inner())
3132 .presented(connection, current_nonce.as_deref())
3133 };
3134 let mut table = self
3139 .scopes
3140 .write()
3141 .unwrap_or_else(|poisoned| poisoned.into_inner());
3142 let outcome = match end {
3143 Some(end) => table.apply(
3144 &owner,
3145 connection_id,
3146 is_current_launch,
3147 generation,
3148 scopes,
3149 end,
3150 ),
3151 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3152 };
3153 let drained = match &outcome {
3154 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3155 Err(_) => Vec::new(),
3156 };
3157 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3158 drop(table);
3159 if has_deadlines {
3160 self.scope_deadline_added.notify_one();
3161 }
3162 match outcome {
3163 Ok(applied) => {
3164 let counts = ScopeOutcomeCounts::of(&applied.results);
3165 info!(
3166 owner = %owner,
3167 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3168 generation,
3169 records = applied.results.len(),
3170 created = counts.created,
3171 replaced = counts.replaced,
3172 updated = counts.updated,
3173 unchanged = counts.unchanged,
3174 refused = counts.refused,
3175 ended = applied.ended.len(),
3176 tag_changes = applied.tag_changes.len(),
3177 routes_closed = drained.len(),
3178 "scope change accepted"
3179 );
3180 for refused in applied
3186 .results
3187 .iter()
3188 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3189 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3190 {
3191 warn!(
3192 owner = %owner,
3193 generation,
3194 scope_ref = %refused.scope_ref,
3195 scope_epoch = refused.scope_epoch,
3196 code = refused.code.as_deref().unwrap_or(""),
3197 "scope record refused"
3198 );
3199 }
3200 self.close_scope_drained_routes(drained);
3201 let response = if is_apply {
3202 ModuleControlResponseToModule::ScopeApply {
3203 generation,
3204 results: applied.results,
3205 end_results: applied.end_results,
3206 ended: applied.ended,
3207 }
3208 } else {
3209 ModuleControlResponseToModule::ScopeSync {
3210 generation,
3211 results: applied.results,
3212 ended: applied.ended,
3213 }
3214 };
3215 Ok(vec![control_response_body_frame(
3216 &frame,
3217 &response,
3218 "scope change response",
3219 )?])
3220 }
3221 Err(refusal) => {
3222 info!(
3223 owner = %owner,
3224 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3225 generation,
3226 code = refusal.code,
3227 "scope change refused"
3228 );
3229 Ok(vec![control_error_frame(
3230 &frame,
3231 refusal.code,
3232 refusal.message,
3233 )?])
3234 }
3235 }
3236 }
3237
3238 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3244 if drained.is_empty() {
3245 return;
3246 }
3247 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3248 BTreeMap::new();
3249 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3250 for route in drained {
3251 warn!(
3252 module_id = %route.module_id,
3253 reason = ?route.reason,
3254 client_connection_id = route.client.connection_id.get(),
3255 route_channel = route.client.channel,
3256 "closing route because its scope changed"
3257 );
3258 pushes
3259 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3260 .or_insert_with(|| (route.reason, Vec::new()))
3261 .1
3262 .push(EndpointRoute {
3263 goodbye_target: route.client.clone(),
3264 principal: Principal::Unverified,
3265 bound_at: Instant::now(),
3266 draining: false,
3267 drain_reason: None,
3268 });
3269 goodbyes.push(route.module);
3270 goodbyes.push(route.client);
3271 }
3272 for ((module_id, _), (reason, routes)) in pushes {
3273 send_route_control_pushes(
3274 &self.forwarding,
3275 routes,
3276 ClientControlPush::RouteClosed {
3277 module_id,
3278 channels: Vec::new(),
3279 reason,
3280 drained: false,
3281 abandoned: 0,
3282 excluded_subscriptions: 0,
3283 terminal: Some(false),
3284 },
3285 );
3286 }
3287 self.emit_route_goodbyes(goodbyes);
3288 }
3289
3290 fn handle_scope_describe(
3293 &self,
3294 connection_id: ConnectionId,
3295 frame: Frame,
3296 owner: Principal,
3297 scope_ref: String,
3298 ) -> Result<Vec<Frame>, RouterError> {
3299 let registered = self
3300 .registry
3301 .get_module_by_connection(connection_id)
3302 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3303 let Some(registered) = registered else {
3304 return Ok(vec![control_error_frame(
3305 &frame,
3306 "not_registered",
3307 "scope.describe requires an active module registration owned by this connection",
3308 )?]);
3309 };
3310 let mut description = self
3311 .scopes
3312 .read()
3313 .unwrap_or_else(|poisoned| poisoned.into_inner())
3314 .describe(&owner, &scope_ref);
3315 if !registered
3318 .manifest
3319 .capabilities
3320 .as_ref()
3321 .is_some_and(|caps| {
3322 caps.provides
3323 .iter()
3324 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3325 })
3326 {
3327 if let Some(stamp) = description.stamp.as_mut() {
3328 stamp.attributes.on_behalf_of_agent = None;
3329 }
3330 }
3331 let owner_configured = match &owner {
3332 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3337 _ => false,
3338 };
3339 let response = ModuleControlResponseToModule::ScopeDescribe {
3340 status: description.status,
3341 scope_epoch: description.scope_epoch,
3342 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3343 owner_synced: description.owner_synced,
3344 owner_configured,
3345 scope: description.stamp,
3346 };
3347 Ok(vec![control_response_body_frame(
3348 &frame,
3349 &response,
3350 "ModuleControlResponseToModule::ScopeDescribe",
3351 )?])
3352 }
3353
3354 fn handle_catalog_update(
3355 &self,
3356 connection_id: ConnectionId,
3357 frame: Frame,
3358 provides: Vec<ProviderRole>,
3359 capabilities: Option<CapabilityDeclarations>,
3360 ready: Option<bool>,
3361 ) -> Result<Vec<Frame>, RouterError> {
3362 self.refresh_capability_requirements();
3363 let Some(registration) = self
3364 .registry
3365 .get_module_by_connection(connection_id)
3366 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3367 else {
3368 return Ok(vec![control_error_frame(
3369 &frame,
3370 "not_registered",
3371 "catalog.update requires an active module registration owned by this connection",
3372 )?]);
3373 };
3374
3375 if let Some(message) =
3376 catalog_update_frozen_field_message(®istration.manifest, &provides)
3377 {
3378 return Ok(vec![control_error_frame(
3379 &frame,
3380 "catalog_update_frozen_field",
3381 message,
3382 )?]);
3383 }
3384
3385 let mut candidate = registration.manifest.clone();
3386 candidate.provides = provides.clone();
3387 candidate.capabilities = capabilities
3388 .clone()
3389 .or_else(|| registration.manifest.capabilities.clone());
3390 if let Err(err) = candidate.validate_capability_grammar() {
3391 return Ok(vec![control_error_frame(
3392 &frame,
3393 "invalid_capability_grammar",
3394 err.to_string(),
3395 )?]);
3396 }
3397
3398 let mut conflicts = self
3401 .capability_evaluator
3402 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3403 if let Some(conflict) = conflicts.first() {
3404 let message = format!(
3405 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3406 conflict.capability, conflict.claimants[0], candidate.module_id
3407 );
3408 for conflict in &mut conflicts {
3409 conflict.source = DuplicateClaimSource::CatalogUpdate;
3410 }
3411 log_duplicate_claim_events(conflicts);
3412 return Ok(vec![control_error_frame(
3413 &frame,
3414 "reserved_capability",
3415 message,
3416 )?]);
3417 }
3418
3419 let updated = self
3420 .registry
3421 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3422 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3423 if updated.is_none() {
3424 return Ok(vec![control_error_frame(
3425 &frame,
3426 "not_registered",
3427 "catalog.update requires an active module registration owned by this connection",
3428 )?]);
3429 }
3430 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3431 log_duplicate_claim_events(
3432 self.capability_evaluator
3433 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3434 );
3435 }
3436 if capability_census_trigger(
3437 registration.manifest.capabilities.as_ref(),
3438 updated
3439 .as_ref()
3440 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3441 ) {
3442 self.enforce_capability_denies();
3443 }
3444 self.refresh_capability_requirements();
3445
3446 let response = ModuleControlResponseToModule::CatalogUpdate {};
3447 control_response_body_frame(
3448 &frame,
3449 &response,
3450 "ModuleControlResponseToModule::CatalogUpdate",
3451 )
3452 .map(|frame| vec![frame])
3453 }
3454
3455 pub(crate) fn with_activity_ledger_config(
3456 mut self,
3457 run_dir: &Path,
3458 incarnation: String,
3459 config: crate::activity::ActivityConfig,
3460 ) -> Self {
3461 self.activity = Some(crate::activity::ActivityLedger::open_config(
3462 run_dir.join("activity.jsonl"),
3463 incarnation,
3464 config,
3465 ));
3466 self.supervisor.set_activity(self.activity.clone());
3467 self
3468 }
3469
3470 pub(crate) async fn configure_activity(
3471 &self,
3472 config: crate::activity::ActivityConfig,
3473 ) -> Result<(), &'static str> {
3474 match &self.activity {
3475 Some(ledger) => ledger.configure(config).await,
3476 None => Ok(()),
3477 }
3478 }
3479
3480 fn cancel_activity_subscription(
3481 &self,
3482 ctx: &RouteCtx,
3483 frame: &Frame,
3484 ) -> Result<bool, RouterError> {
3485 let task = self
3486 .activity_streams
3487 .lock()
3488 .unwrap_or_else(|p| p.into_inner())
3489 .remove(&(ctx.connection_id, frame.header.corr));
3490 let Some(task) = task else {
3491 return Ok(false);
3492 };
3493 task.abort();
3494 let end = Frame::build_with_version(
3495 response_version(frame),
3496 FrameType::StreamEnd,
3497 control_flags(),
3498 0,
3499 0,
3500 frame.header.corr,
3501 Vec::new(),
3502 )
3503 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3504 let sink = ctx.egress.clone();
3505 tokio::spawn(async move {
3506 let _ = sink.send(end).await;
3507 });
3508 Ok(true)
3509 }
3510
3511 async fn handle_activity_read(
3512 &self,
3513 ctx: &RouteCtx,
3514 frame: Frame,
3515 ) -> Result<Vec<Frame>, RouterError> {
3516 use subc_control::{
3517 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3518 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3519 };
3520 #[derive(Deserialize)]
3521 struct Identity {
3522 #[serde(default)]
3523 consumer_identity: Option<ConsumerIdentity>,
3524 }
3525 let principal = match self
3529 .forwarding
3530 .module_id_for_connection(ctx.connection_id)
3531 .map_err(RouterError::Forwarding)?
3532 {
3533 Some(module_id) => Ok(Principal::Reserved { module_id }),
3534 None => {
3535 let identity: Identity = match serde_json::from_slice(&frame.body) {
3536 Ok(identity) => identity,
3537 Err(_) => {
3538 return Ok(vec![control_error_frame(
3539 &frame,
3540 ACTIVITY_MALFORMED,
3541 "malformed activity read",
3542 )?])
3543 }
3544 };
3545 self.route_open_principal(&frame, identity.consumer_identity)?
3546 }
3547 };
3548 let admitted = matches!(principal, Ok(Principal::Direct))
3549 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3550 if !admitted {
3551 return Ok(vec![control_error_frame(
3552 &frame,
3553 ACTIVITY_READ_FORBIDDEN,
3554 "activity reads require an operator or trusted mirror principal",
3555 )?]);
3556 }
3557 let Some(ledger) = &self.activity else {
3558 return Ok(vec![control_error_frame(
3559 &frame,
3560 ACTIVITY_UNAVAILABLE,
3561 "activity ledger unavailable",
3562 )?]);
3563 };
3564 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3565 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3566 {
3567 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3568 Ok(request) => ledger.query(request).await,
3569 Err(_) => Err(ACTIVITY_MALFORMED),
3570 };
3571 return match result {
3572 Ok(reply) => Ok(vec![control_response_body_frame(
3573 &frame,
3574 &reply,
3575 ops::ACTIVITY_QUERY,
3576 )?]),
3577 Err(code) => Ok(vec![control_error_frame(
3578 &frame,
3579 code,
3580 "activity query refused",
3581 )?]),
3582 };
3583 }
3584 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3585 Ok(request) => request,
3586 Err(_) => {
3587 return Ok(vec![control_error_frame(
3588 &frame,
3589 ACTIVITY_MALFORMED,
3590 "malformed activity subscription",
3591 )?])
3592 }
3593 };
3594 let mut subscription = match ledger.subscribe(request.since).await {
3595 Ok(subscription) => subscription,
3596 Err(code) => {
3597 return Ok(vec![control_error_frame(
3598 &frame,
3599 code,
3600 "activity subscription refused",
3601 )?])
3602 }
3603 };
3604 let key = (ctx.connection_id, frame.header.corr);
3605 let sink = ctx.egress.clone();
3606 let mut streams = self
3607 .activity_streams
3608 .lock()
3609 .unwrap_or_else(|p| p.into_inner());
3610 streams.retain(|_, task| !task.is_finished());
3611 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3612 return Ok(vec![control_error_frame(
3613 &frame,
3614 "duplicate_subscription",
3615 "activity subscription correlation id already in use",
3616 )?]);
3617 }
3618 let task = tokio::spawn(async move {
3619 loop {
3620 let entry = if let Some(entry) = subscription.replay.pop_front() {
3621 Some(entry)
3622 } else {
3623 subscription.receiver.recv().await
3624 };
3625 let Some(entry) = entry else {
3626 break;
3627 };
3628 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3629 return;
3630 };
3631 let Ok(next) = Frame::build_with_version(
3632 response_version(&frame),
3633 FrameType::StreamData,
3634 control_flags(),
3635 0,
3636 0,
3637 frame.header.corr,
3638 body,
3639 ) else {
3640 return;
3641 };
3642 if sink.send(next).await.is_err() {
3643 return;
3644 }
3645 }
3646 if let Some(code) = subscription.terminal_code() {
3649 if let Ok(end) =
3650 control_error_frame(&frame, code, "activity subscriber fell behind")
3651 {
3652 let _ = sink.send(end).await;
3653 }
3654 }
3655 });
3656 streams.insert(key, task.abort_handle());
3657 Ok(Vec::new())
3658 }
3659
3660 async fn handle_activity_record(
3661 &self,
3662 ctx: &RouteCtx,
3663 frame: Frame,
3664 ) -> Result<Vec<Frame>, RouterError> {
3665 use subc_control::{
3666 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3667 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3668 };
3669 let result = async {
3670 let module = self
3671 .forwarding
3672 .module_id_for_connection(ctx.connection_id)
3673 .ok()
3674 .flatten()
3675 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3676 let request: ActivityRecordRequest =
3677 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3678 crate::activity::validate(&module, &request)?;
3679 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3680 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3681 let at_ms = std::time::SystemTime::now()
3682 .duration_since(std::time::UNIX_EPOCH)
3683 .unwrap_or_default()
3684 .as_millis() as u64;
3685 ledger.record(module, request, actor, at_ms).await
3686 }
3687 .await;
3688 match result {
3689 Ok(entry) => Ok(vec![control_response_body_frame(
3690 &frame,
3691 &ActivityRecordResponse { seq: entry.seq },
3692 "activity.record",
3693 )?]),
3694 Err(code) => Ok(vec![control_error_frame(
3695 &frame,
3696 code,
3697 "activity recording refused",
3698 )?]),
3699 }
3700 }
3701
3702 fn activity_actor(
3703 &self,
3704 connection: ConnectionId,
3705 module: &str,
3706 on: Option<&subc_control::ActivityOn>,
3707 ) -> Result<subc_control::ActivityActor, &'static str> {
3708 use crate::forwarding::{DataRoute, DataRouteState};
3709 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3710 let principal = |id: &str| Principal::Reserved {
3711 module_id: id.to_owned(),
3712 };
3713 let mut actor = ActivityActor {
3714 principal: principal(module),
3715 scope_owner: None,
3716 agent_id: None,
3717 run_id: None,
3718 flow_id: None,
3719 project_root: None,
3720 };
3721 let scope = match on {
3722 None => return Ok(actor),
3723 Some(ActivityOn::Scope(scope)) => {
3724 let selector = subc_protocol::scope::ScopeSelector::new(
3725 principal(module),
3726 scope.scope_ref.clone(),
3727 Some(scope.epoch),
3728 );
3729 self.scopes
3730 .read()
3731 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3732 .attribute(&principal(module), module, &selector, true)
3733 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3734 }
3735 Some(ActivityOn::Route(route)) => {
3736 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3737 .forwarding
3738 .lookup_data_route(connection, route.channel, route.epoch)
3739 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3740 else {
3741 return Err(ACTIVITY_ACTOR_UNKNOWN);
3742 };
3743 if binding.module_id != module {
3744 return Err(ACTIVITY_ACTOR_UNKNOWN);
3745 }
3746 actor.principal = binding.principal.clone();
3747 actor.project_root = binding
3748 .project_root
3749 .as_ref()
3750 .map(|root| root.as_path().to_path_buf());
3751 let Some(bound) = &binding.scope else {
3752 return Ok(actor);
3753 };
3754 let owner = principal(&bound.owner);
3755 let selector = subc_protocol::scope::ScopeSelector::new(
3756 owner.clone(),
3757 bound.scope_ref.clone(),
3758 Some(bound.tag.scope_epoch),
3759 );
3760 let admitted = self
3761 .scopes
3762 .read()
3763 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3764 .attribute(&owner, module, &selector, true)
3765 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3766 if admitted.tag != bound.tag {
3767 return Err(ACTIVITY_ACTOR_UNKNOWN);
3768 }
3769 admitted
3770 }
3771 };
3772 actor.scope_owner = Some(scope.stamp.owner);
3773 actor.agent_id = scope.stamp.attributes.agent_id;
3774 actor.run_id = scope.stamp.attributes.run_id;
3775 actor.flow_id = scope.stamp.attributes.flow_id;
3776 Ok(actor)
3777 }
3778
3779 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3780 self.refresh_capability_requirements();
3781 let mut counters = self.counters.snapshot();
3787 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3788 obj.insert(
3789 "activity_unparseable_lines".into(),
3790 ledger.read_errors().into(),
3791 );
3792 obj.insert(
3793 "activity_lifecycle_append_failures".into(),
3794 ledger.lifecycle_failures().into(),
3795 );
3796 obj.insert(
3797 "activity_started_digest_skips".into(),
3798 ledger.lifecycle_digest_skips().into(),
3799 );
3800 }
3801 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3802 self.forwarding.client_route_concentration(),
3803 counters.as_object_mut(),
3804 ) {
3805 obj.insert(
3806 "client_connections_with_routes".into(),
3807 connections_with_routes.into(),
3808 );
3809 obj.insert("max_routes_on_one_connection".into(), max.into());
3810 }
3811 if let (Some(open_breakers), Some(obj)) = (
3817 self.route_bind_breakers.open_snapshot(),
3818 counters.as_object_mut(),
3819 ) {
3820 obj.insert("route_bind_breakers_open".into(), open_breakers);
3821 }
3822 let response = ClientControlResponse::ServerDescribe {
3823 protocol_ver: PROTOCOL_VERSION,
3824 subc_ops: subc_ops(),
3825 capabilities: self.subc_capabilities.as_ref().to_vec(),
3826 connected_clients: self.connected_clients.count(),
3827 counters: Some(counters),
3828 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3829 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3830 capability_requirements: self.capability_requirement_statuses(),
3831 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3832 };
3833 Ok(vec![control_response_body_frame(
3834 &frame,
3835 &response,
3836 "ClientControlResponse::ServerDescribe",
3837 )?])
3838 }
3839
3840 fn handle_catalog_list(
3841 &self,
3842 frame: Frame,
3843 module_id: Option<String>,
3844 ) -> Result<Vec<Frame>, RouterError> {
3845 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3846 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3847 })?;
3848 let entries = modules
3849 .into_iter()
3850 .filter(|registration| {
3851 module_id
3852 .as_deref()
3853 .map(|wanted| registration.manifest.module_id == wanted)
3854 .unwrap_or(true)
3855 })
3856 .map(|registration| {
3857 let not_ready = self.not_ready_reason(®istration);
3858 let roles = registration.manifest.provides;
3859 CatalogEntry::new(
3860 registration.manifest.module_id,
3861 roles,
3862 registration.control_ops,
3863 )
3864 .with_ready(not_ready.is_none())
3865 .with_not_ready(not_ready)
3866 .with_module_version(Some(registration.manifest.module_version))
3867 .with_capabilities(registration.manifest.capabilities)
3868 .with_self_signals(registration.manifest.self_signals)
3869 .with_events(registration.manifest.events)
3870 .with_operation_delivery(registration.manifest.operation_delivery)
3871 })
3872 .collect();
3873 let response = ClientControlResponse::CatalogList {
3874 generation,
3875 modules: entries,
3876 subc_ops: subc_ops(),
3877 };
3878 Ok(vec![control_response_body_frame(
3879 &frame,
3880 &response,
3881 "ClientControlResponse::CatalogList",
3882 )?])
3883 }
3884
3885 fn route_open_principal(
3886 &self,
3887 frame: &Frame,
3888 consumer_identity: Option<ConsumerIdentity>,
3889 ) -> Result<Result<Principal, Frame>, RouterError> {
3890 let Some(consumer_identity) = consumer_identity else {
3891 return Ok(Ok(Principal::Direct));
3892 };
3893
3894 if self.supervisor.spawned_consumer_authorized(
3895 &consumer_identity.module_id,
3896 &consumer_identity.launch_nonce,
3897 ) {
3898 return Ok(Ok(Principal::Reserved {
3899 module_id: consumer_identity.module_id,
3900 }));
3901 }
3902
3903 Ok(Err(control_error_frame(
3904 frame,
3905 "bad_consumer_identity",
3906 format!(
3907 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3908 consumer_identity.module_id
3909 ),
3910 )?))
3911 }
3912
3913 fn route_open_refusal_frame(
3920 &self,
3921 ctx: &RouteCtx,
3922 frame: &Frame,
3923 module_id: &str,
3924 reason: &'static str,
3925 code: &'static str,
3926 message: impl Into<String>,
3927 ) -> Result<Frame, RouterError> {
3928 self.observe_route_open_refusal(ctx, module_id, reason, code);
3929 control_error_frame(frame, code, message.into())
3930 }
3931
3932 fn route_open_breaker_refusal_frame(
3950 &self,
3951 ctx: &RouteCtx,
3952 frame: &Frame,
3953 module_id: &str,
3954 consecutive_timeouts: u32,
3955 retry_in: Duration,
3956 probe_in_flight: bool,
3957 ) -> Result<Frame, RouterError> {
3958 self.counters
3959 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3960 debug!(
3961 target: "control",
3962 code = "module_timeout",
3963 module_id = ?module_id,
3964 connection_id = ctx.connection_id.get(),
3965 consecutive_timeouts,
3966 retry_in_ms = retry_in.as_millis() as u64,
3967 probe_in_flight,
3968 "route.open refused by open bind-relay breaker"
3969 );
3970 let detail = if probe_in_flight {
3977 "one test route is already being tried; retry once it settles".to_string()
3978 } else {
3979 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3980 };
3981 control_error_frame(
3982 frame,
3983 "module_timeout",
3984 format!(
3985 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3986 timed out in a row); {detail}; its established routes are unaffected"
3987 ),
3988 )
3989 }
3990
3991 fn observe_route_open_refusal(
4004 &self,
4005 ctx: &RouteCtx,
4006 module_id: &str,
4007 reason: &'static str,
4008 code: &'static str,
4009 ) {
4010 self.counters.increment_route_open_refused(code);
4011 info!(
4012 target: "control",
4013 code,
4014 reason,
4015 module_id = ?module_id,
4016 connection_id = ctx.connection_id.get(),
4017 "route.open refused"
4018 );
4019 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4020 self.route_outages.record_not_serving(module_id, reason);
4021 }
4022 }
4023
4024 fn observe_route_open_accept(
4069 &self,
4070 ctx: &RouteCtx,
4071 module_id: &str,
4072 principal: &str,
4073 scope: &str,
4074 ) {
4075 self.route_outages.record_accepted(module_id);
4076 self.counters.increment_route_open_accepted(principal);
4077 info!(
4078 target: "control",
4079 principal,
4080 module_id,
4081 connection_id = ctx.connection_id.get(),
4082 scope = %scope,
4083 "route.open accepted"
4084 );
4085 }
4086
4087 fn supervised_absent_route_open_refusal_frame(
4088 &self,
4089 ctx: &RouteCtx,
4090 frame: &Frame,
4091 module_id: &str,
4092 code: &'static str,
4093 status: &crate::supervise::ModuleStatus,
4094 ) -> Result<Frame, RouterError> {
4095 self.counters.increment_route_open_refused(code);
4096 info!(
4097 target: "control",
4098 code,
4099 reason = "supervised_not_registered",
4100 module_id = ?module_id,
4101 connection_id = ctx.connection_id.get(),
4102 state = %status.state,
4103 enabled = status.enabled,
4104 live = status.live,
4105 "route.open refused"
4106 );
4107 self.route_outages
4111 .record_not_serving(module_id, "supervised_not_registered");
4112 let message = if let Some(pid) = status.waiting_for_previous_pid {
4113 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4114 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4115 })
4116 } else {
4117 format!(
4118 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4119 status.state, status.enabled, status.live
4120 )
4121 };
4122 control_error_frame(frame, code, message)
4123 }
4124
4125 async fn handle_route_open(
4126 &self,
4127 ctx: &RouteCtx,
4128 frame: Frame,
4129 request: RouteOpenRequest,
4130 timings: &mut RouteOpenTimings,
4131 ) -> Result<Vec<Frame>, RouterError> {
4132 let RouteOpenRequest {
4133 target,
4134 mut identity,
4135 consumer_identity,
4136 consumer_capabilities,
4137 role_versions,
4138 admission_facts,
4139 scope,
4140 } = request;
4141 let target_module_id = target_module_id(&target).to_string();
4142 timings.module_id.clone_from(&target_module_id);
4143 if self
4144 .registry
4145 .get_module_by_connection(ctx.connection_id)
4146 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4147 .is_some()
4148 {
4149 return Ok(vec![control_error_frame(
4150 &frame,
4151 "invalid_request",
4152 "module connections cannot open client routes",
4153 )?]);
4154 }
4155 debug!(
4156 connection_id = ctx.connection_id.get(),
4157 corr = frame.header.corr,
4158 module_id = %target_module_id,
4159 "handling route.open"
4160 );
4161
4162 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4168 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4169 self.observe_route_open_refusal(
4170 ctx,
4171 &target_module_id,
4172 "invalid_role_versions",
4173 error_codes::INVALID_REQUEST,
4174 );
4175 return Ok(vec![control_error_body_frame(
4176 &frame,
4177 ErrorBody {
4178 code: error_codes::INVALID_REQUEST.to_string(),
4179 message: error.to_string(),
4180 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4181 },
4182 )?]);
4183 }
4184
4185 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4186 if status.waiting_for_previous_pid.is_some() {
4187 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4188 ctx,
4189 &frame,
4190 &target_module_id,
4191 "module_warming",
4192 &status,
4193 )?]);
4194 }
4195 }
4196
4197 let Some(registration) = self
4216 .registry
4217 .get_module(&target_module_id)
4218 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4219 else {
4220 if let Some((status, warming)) =
4221 self.supervisor_status(&target_module_id, frame.header.corr)?
4222 {
4223 if status.protocol == ModuleProtocol::None {
4230 return Ok(vec![self.route_open_refusal_frame(
4231 ctx,
4232 &frame,
4233 &target_module_id,
4234 "protocol_none",
4235 error_codes::MODULE_NO_PROTOCOL,
4236 format!(
4237 "module_id '{target_module_id}' is declared protocol: none; \
4238 it speaks no subc wire and serves no routes"
4239 ),
4240 )?]);
4241 }
4242 let code = if warming {
4243 "module_warming"
4244 } else {
4245 "target_unavailable"
4246 };
4247 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4248 ctx,
4249 &frame,
4250 &target_module_id,
4251 code,
4252 &status,
4253 )?]);
4254 }
4255 if let Some(removed_ago_ms) =
4256 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4257 {
4258 return Ok(vec![self.route_open_refusal_frame(
4259 ctx,
4260 &frame,
4261 &target_module_id,
4262 "removed",
4263 error_codes::MODULE_REMOVED,
4264 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4265 )?]);
4266 }
4267 return Ok(vec![self.route_open_refusal_frame(
4268 ctx,
4269 &frame,
4270 &target_module_id,
4271 "not_registered",
4272 error_codes::UNKNOWN_MODULE,
4273 format!("module_id '{target_module_id}' is not registered"),
4274 )?]);
4275 };
4276
4277 if !registration.ready {
4281 self.counters
4282 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4283 info!(
4284 target: "control",
4285 code = error_codes::MODULE_WARMING,
4286 module_id = ?target_module_id,
4287 connection_id = ctx.connection_id.get(),
4288 reason = "declared_not_ready",
4289 "route.open refused"
4290 );
4291 self.route_outages
4294 .record_not_serving(&target_module_id, "declared_not_ready");
4295 return Ok(vec![control_error_body_frame(
4296 &frame,
4297 ErrorBody {
4298 code: error_codes::MODULE_WARMING.to_string(),
4299 message: format!(
4300 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4301 ),
4302 detail: Some(serde_json::json!({
4303 "reason": "declared_not_ready"
4304 })),
4305 },
4306 )?]);
4307 }
4308
4309 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4328 self.counters
4329 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4330 info!(
4331 target: "control",
4332 code = error_codes::MODULE_WARMING,
4333 module_id = ?target_module_id,
4334 connection_id = ctx.connection_id.get(),
4335 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4336 capability = %capability,
4337 "route.open refused"
4338 );
4339 return Ok(vec![control_error_body_frame(
4340 &frame,
4341 ErrorBody {
4342 code: error_codes::MODULE_WARMING.to_string(),
4343 message: format!(
4344 "module_id '{target_module_id}' requires capability '{capability}', \
4345 which no registered module provides; retry"
4346 ),
4347 detail: Some(serde_json::json!({
4348 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4349 "capability": capability,
4350 })),
4351 },
4352 )?]);
4353 }
4354
4355 if !target_has_required_role(&target, ®istration.manifest.provides) {
4356 return Ok(vec![self.route_open_refusal_frame(
4357 ctx,
4358 &frame,
4359 &target_module_id,
4360 "role_not_provided",
4361 "target_unavailable",
4362 format!("module_id '{target_module_id}' does not provide the requested target"),
4363 )?]);
4364 }
4365
4366 if registration.state != ChannelState::Active {
4367 return Ok(vec![self.route_open_refusal_frame(
4368 ctx,
4369 &frame,
4370 &target_module_id,
4371 "registration_not_active",
4372 "target_unavailable",
4373 format!("module_id '{target_module_id}' is not active"),
4374 )?]);
4375 }
4376
4377 if self
4378 .forwarding
4379 .module_is_draining(&target_module_id)
4380 .map_err(RouterError::Forwarding)?
4381 {
4382 return Ok(vec![self.route_open_refusal_frame(
4383 ctx,
4384 &frame,
4385 &target_module_id,
4386 "reloading",
4387 "module_reloading",
4388 format!("module_id '{target_module_id}' is reloading"),
4389 )?]);
4390 }
4391
4392 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4393 process_liveness.process_live(&target_module_id) == Some(false)
4394 }) {
4395 if process_liveness.process_replacing(&target_module_id) {
4402 return Ok(vec![self.route_open_refusal_frame(
4403 ctx,
4404 &frame,
4405 &target_module_id,
4406 "reloading",
4407 "module_reloading",
4408 format!("module_id '{target_module_id}' is reloading"),
4409 )?]);
4410 }
4411 return Ok(vec![self.route_open_refusal_frame(
4412 ctx,
4413 &frame,
4414 &target_module_id,
4415 "supervisor_not_live",
4416 "target_unavailable",
4417 format!("module_id '{target_module_id}' is not live"),
4418 )?]);
4419 }
4420
4421 if !self
4422 .forwarding
4423 .has_live_module_connection(&target_module_id)
4424 .map_err(RouterError::Forwarding)?
4425 {
4426 return Ok(vec![self.route_open_refusal_frame(
4427 ctx,
4428 &frame,
4429 &target_module_id,
4430 "no_forwarding_connection",
4431 "target_unavailable",
4432 format!("module_id '{target_module_id}' has no live forwarding connection"),
4433 )?]);
4434 }
4435
4436 if let Some(error) =
4437 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4438 {
4439 self.observe_route_open_refusal(
4440 ctx,
4441 &target_module_id,
4442 "op_not_allowed",
4443 "op_not_allowed",
4444 );
4445 return Ok(vec![error]);
4446 }
4447
4448 let principal = match self.route_open_principal(&frame, consumer_identity)? {
4449 Ok(principal) => principal,
4450 Err(error) => {
4451 self.observe_route_open_refusal(
4452 ctx,
4453 &target_module_id,
4454 "bad_consumer_identity",
4455 "bad_consumer_identity",
4456 );
4457 return Ok(vec![error]);
4458 }
4459 };
4460 timings.principal = principal_label(&principal);
4461
4462 if let Principal::Reserved {
4466 module_id: opening_module_id,
4467 } = &principal
4468 {
4469 if let Some(opening_registration) = self
4470 .registry
4471 .get_module(opening_module_id)
4472 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4473 {
4474 if let Some(capability) =
4475 denied_capability(&opening_registration.manifest, ®istration.manifest)
4476 {
4477 warn!(
4478 opening_module_id,
4479 target_module_id,
4480 capability,
4481 "refusing route.open because an attested capability deny edge matches"
4482 );
4483 return Ok(vec![self.route_open_refusal_frame(
4484 ctx,
4485 &frame,
4486 &target_module_id,
4487 "capability_deny_edge",
4488 "capability_forbidden",
4489 format!(
4490 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4491 ),
4492 )?]);
4493 }
4494 }
4495 }
4496
4497 if admission_facts.is_some() {
4498 let carrier_matches = matches!(
4499 &principal,
4500 Principal::Reserved { module_id }
4501 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4502 );
4503 if !carrier_matches {
4504 return Ok(vec![self.route_open_refusal_frame(
4505 ctx,
4506 &frame,
4507 &target_module_id,
4508 "admission_facts_carrier_not_permitted",
4509 "admission_facts_not_permitted",
4510 "admission facts may only be carried by the configured reserved module",
4511 )?]);
4512 }
4513
4514 let target_allowed = self
4515 .admission_facts_targets
4516 .as_ref()
4517 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4518 if !target_allowed {
4519 return Ok(vec![self.route_open_refusal_frame(
4520 ctx,
4521 &frame,
4522 &target_module_id,
4523 "admission_facts_target_not_listed",
4524 "admission_facts_target_not_allowed",
4525 format!(
4526 "admission facts are not permitted for target module_id '{target_module_id}'"
4527 ),
4528 )?]);
4529 }
4530
4531 }
4535
4536 let (bound_scope, mut scope_stamp) = match &scope {
4542 None => (None, None),
4543 Some(selector) => {
4544 let owner_configured = match &selector.owner {
4545 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4551 _ => false,
4552 };
4553 let mut scopes = self
4554 .scopes
4555 .write()
4556 .unwrap_or_else(|poisoned| poisoned.into_inner());
4557 let admitted = scopes
4558 .admit(&principal, &target_module_id, selector, owner_configured)
4559 .and_then(|admission| {
4560 crate::scopes::check_target_flow_support(
4561 &admission.stamp,
4562 &target_module_id,
4563 registration.manifest.capabilities.as_ref(),
4564 )?;
4565 crate::scopes::check_target_agent_run_support(
4566 &admission.stamp,
4567 &target_module_id,
4568 registration.manifest.capabilities.as_ref(),
4569 )?;
4570 let bound = scopes.reserve_bind(
4571 &admission,
4572 &principal,
4573 ctx.connection_id,
4574 selector.hold_presence,
4575 );
4576 Ok((admission, bound))
4577 });
4578 match admitted {
4579 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4580 Err(refusal) => {
4581 return Ok(vec![self.route_open_refusal_frame(
4582 ctx,
4583 &frame,
4584 &target_module_id,
4585 refusal.code,
4586 refusal.code,
4587 refusal.message,
4588 )?]);
4589 }
4590 }
4591 }
4592 };
4593
4594 timings.enter(RouteOpenPhase::ProjectRoot);
4616 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4617 Ok(project_root) => project_root,
4618 Err(err) => {
4619 return Ok(vec![control_error_frame(
4620 &frame,
4621 "invalid_project_root",
4622 err.to_string(),
4623 )?])
4624 }
4625 };
4626 timings.enter(RouteOpenPhase::Admission);
4627 identity.project_root = project_root.as_path().to_path_buf();
4628
4629 let _concurrency_guard = match self
4640 .route_bind_concurrency
4641 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4642 {
4643 Ok(guard) => guard,
4644 Err(in_flight) => {
4645 return Ok(vec![self.route_open_target_capacity_refusal(
4646 ctx,
4647 &frame,
4648 &target_module_id,
4649 in_flight,
4650 )?]);
4651 }
4652 };
4653
4654 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4657 RouteBindAdmission::Admitted { guard, probe } => {
4658 if probe {
4659 info!(
4660 module_id = %target_module_id,
4661 connection_id = ctx.connection_id.get(),
4662 "route.bind breaker half-open: admitting one probe"
4663 );
4664 }
4665 guard
4666 }
4667 RouteBindAdmission::Refused {
4668 consecutive_timeouts,
4669 retry_in,
4670 probe_in_flight,
4671 } => {
4672 return Ok(vec![self.route_open_breaker_refusal_frame(
4673 ctx,
4674 &frame,
4675 &target_module_id,
4676 consecutive_timeouts,
4677 retry_in,
4678 probe_in_flight,
4679 )?]);
4680 }
4681 };
4682
4683 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4688 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4689 timings.enter(RouteOpenPhase::EgressReservation);
4690 let pending = match self
4691 .forwarding
4692 .begin_route_bind_relay_for(
4693 ctx.connection_id,
4694 ctx.egress.clone(),
4695 response_version(&frame),
4696 frame.header.corr,
4697 &target_module_id,
4698 principal.clone(),
4699 bound_scope,
4700 Some(project_root),
4701 relay_deadline,
4702 )
4703 .await
4704 {
4705 Ok(pending) => pending,
4706 Err(err) => {
4707 return Ok(vec![self.route_open_refusal_frame(
4708 ctx,
4709 &frame,
4710 &target_module_id,
4711 "relay_reservation_failed",
4712 forwarding_error_code(&err),
4713 err.to_string(),
4714 )?])
4715 }
4716 };
4717 timings.enter(RouteOpenPhase::Admission);
4718 let crate::forwarding::PendingRouteBindRelay {
4719 endpoint,
4720 module_sink,
4721 negotiated_ver,
4722 client_channel,
4723 client_epoch,
4724 module_channel,
4725 module_epoch,
4726 corr: relay_corr,
4727 receiver,
4728 } = pending;
4729 let mut reservation =
4730 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4731
4732 let relay_registration = self
4737 .registry
4738 .get_module_by_connection(endpoint.connection_id)
4739 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4740 let relay_capabilities = relay_registration
4741 .as_ref()
4742 .and_then(|registration| registration.manifest.capabilities.as_ref());
4743 if let Some(selector) = &scope {
4744 if let Err(refusal) = self
4747 .scopes
4748 .read()
4749 .unwrap_or_else(|poisoned| poisoned.into_inner())
4750 .check_bind_proof(&principal, selector)
4751 {
4752 reservation.release_and_disarm();
4753 return Ok(vec![self.route_open_refusal_frame(
4754 ctx,
4755 &frame,
4756 &target_module_id,
4757 refusal.code,
4758 refusal.code,
4759 refusal.message,
4760 )?]);
4761 }
4762 }
4763 if let Some(stamp) = scope_stamp
4764 .as_ref()
4765 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4766 {
4767 if let Err(refusal) = crate::scopes::check_target_flow_support(
4768 stamp,
4769 &target_module_id,
4770 relay_capabilities,
4771 )
4772 .and_then(|()| {
4773 crate::scopes::check_target_agent_run_support(
4774 stamp,
4775 &target_module_id,
4776 relay_capabilities,
4777 )
4778 }) {
4779 reservation.release_and_disarm();
4780 return Ok(vec![self.route_open_refusal_frame(
4781 ctx,
4782 &frame,
4783 &target_module_id,
4784 refusal.code,
4785 refusal.code,
4786 refusal.message,
4787 )?]);
4788 }
4789 }
4790
4791 if !relay_capabilities.is_some_and(|caps| {
4794 caps.provides
4795 .iter()
4796 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
4797 }) {
4798 if let Some(stamp) = scope_stamp.as_mut() {
4799 stamp.attributes.on_behalf_of_agent = None;
4800 }
4801 }
4802 let call_mode = if relay_capabilities
4803 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4804 {
4805 Some(derive_call_mode(scope_stamp.as_ref()))
4806 } else {
4807 if let Some(stamp) = scope_stamp.as_mut() {
4808 stamp.expires_at_ms = None;
4809 }
4810 None
4811 };
4812
4813 debug!(
4814 connection_id = ctx.connection_id.get(),
4815 client_channel,
4816 client_epoch,
4817 module_channel,
4818 module_epoch,
4819 "reserved route handle pair"
4820 );
4821 let route_principal_label = principal_label(&principal);
4824 let scope_label = scope_stamp
4825 .as_ref()
4826 .map_or_else(|| "none".to_string(), route_open_scope_label);
4827 let relay = ModuleControlRequest::RouteBind {
4828 route_channel: module_channel,
4829 epoch: module_epoch,
4830 target,
4831 identity,
4832 principal: Some(principal),
4833 consumer_capabilities,
4834 role_versions,
4835 admission_facts,
4836 scope: scope_stamp,
4837 call_mode,
4838 };
4839 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4840 RouterError::backend(
4841 0,
4842 frame.header.corr,
4843 format!("failed to encode route.bind request: {err}"),
4844 )
4845 })?;
4846 let relay_frame = Frame::build_with_version(
4847 negotiated_ver,
4848 FrameType::Request,
4849 control_flags(),
4850 0,
4851 0,
4852 relay_corr,
4853 relay_body,
4854 )
4855 .map_err(RouterError::FrameBuild)?;
4856
4857 timings.enter(RouteOpenPhase::BindSend);
4858 let sent = module_sink.send(relay_frame).await;
4859 timings.enter(RouteOpenPhase::Admission);
4860 if let Err(err) = sent {
4861 reservation.release_and_disarm();
4862 return Ok(vec![self.route_open_refusal_frame(
4863 ctx,
4864 &frame,
4865 &target_module_id,
4866 "relay_send_failed",
4867 "target_unavailable",
4868 err.to_string(),
4869 )?]);
4870 }
4871
4872 #[cfg(test)]
4873 if let Some(hold) = &self.route_bind_relay_sent_hold {
4874 hold.wait().await;
4878 hold.wait().await;
4879 }
4880
4881 if self
4886 .forwarding
4887 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4888 .map_err(RouterError::Forwarding)?
4889 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4890 {
4891 self.send_abandoned_route_bind_goodbye(
4892 &module_sink,
4893 negotiated_ver,
4894 module_channel,
4895 module_epoch,
4896 );
4897 }
4898
4899 timings.enter(RouteOpenPhase::BindAckWait);
4900 let outcome = timeout_at(relay_deadline, receiver).await;
4901 timings.enter(RouteOpenPhase::Admission);
4902 match outcome {
4903 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4904 reservation.disarm();
4905 if breaker.record_accepted() {
4906 info!(
4907 module_id = %target_module_id,
4908 "route.bind breaker closed: the probe was accepted"
4909 );
4910 }
4911 self.observe_route_open_accept(
4912 ctx,
4913 &target_module_id,
4914 &route_principal_label,
4915 &scope_label,
4916 );
4917 Ok(Vec::new())
4918 }
4919 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4920 reservation.release_and_disarm();
4921 breaker.record_inconclusive();
4925 let scope_code = match body.code.as_str() {
4929 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4930 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4931 _ => None,
4932 };
4933 if let Some(code) = scope_code {
4934 self.observe_route_open_refusal(
4935 ctx,
4936 &target_module_id,
4937 "scope_changed_before_commit",
4938 code,
4939 );
4940 return Ok(vec![control_error_body_frame(&frame, body)?]);
4941 }
4942 self.counters
4943 .increment_route_open_refused("module_rejected");
4944 info!(
4945 target: "control",
4946 code = "module_rejected",
4947 module_code = ?body.code,
4948 module_id = ?target_module_id,
4949 connection_id = ctx.connection_id.get(),
4950 "route.open refused"
4951 );
4952 Ok(vec![control_error_body_frame(&frame, body)?])
4953 }
4954 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4955 reservation.release_and_disarm();
4956 breaker.record_inconclusive();
4957 tracing::warn!(
4964 module_id = %target_module_id,
4965 "route.bind relay abandoned: {message}"
4966 );
4967 Ok(vec![self.route_open_refusal_frame(
4968 ctx,
4969 &frame,
4970 &target_module_id,
4971 "relay_abandoned",
4972 "target_unavailable",
4973 message,
4974 )?])
4975 }
4976 Ok(Err(_)) => {
4977 reservation.release_and_disarm();
4978 breaker.record_inconclusive();
4979 Ok(vec![self.route_open_refusal_frame(
4980 ctx,
4981 &frame,
4982 &target_module_id,
4983 "relay_waiter_canceled",
4984 "target_unavailable",
4985 "route.bind relay waiter was canceled before the module responded",
4986 )?])
4987 }
4988 Err(_) => {
4989 reservation.release_and_disarm();
4990 if let Some(opened) = breaker.record_timeout(
4994 self.route_bind_breaker_threshold,
4995 self.route_bind_breaker_cooldown,
4996 ) {
4997 warn!(
4998 module_id = %target_module_id,
4999 consecutive_timeouts = opened.consecutive_timeouts,
5000 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5001 reopened_after_probe = opened.reopened_after_probe,
5002 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5003 );
5004 }
5005 tracing::warn!(
5013 module_id = %target_module_id,
5014 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5015 "route.bind relay timed out: module did not ack within budget"
5016 );
5017 Ok(vec![self.route_open_refusal_frame(
5018 ctx,
5019 &frame,
5020 &target_module_id,
5021 "relay_timed_out",
5022 "module_timeout",
5023 format!(
5024 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5025 route_bind_relay_timeout
5026 ),
5027 )?])
5028 }
5029 }
5030 }
5031
5032 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5033 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5034 snapshot: self.supervisor.spawn_snapshot(),
5035 };
5036 Ok(vec![control_response_body_frame(
5037 &frame,
5038 &response,
5039 "ClientControlResponse::SupervisorSpawnSnapshot",
5040 )?])
5041 }
5042
5043 fn handle_supervisor_spawn_subscribe(
5044 &self,
5045 ctx: &RouteCtx,
5046 frame: Frame,
5047 since: Option<SpawnCursor>,
5048 ) -> Result<Vec<Frame>, RouterError> {
5049 match self.supervisor.subscribe_spawns(
5050 ctx.connection_id,
5051 frame.header.corr,
5052 response_version(&frame),
5053 since,
5054 ctx.egress.clone(),
5055 ) {
5056 Ok(()) => Ok(Vec::new()),
5057 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5058 Ok(vec![control_error_body_frame(
5059 &frame,
5060 ErrorBody {
5061 code: "spawn_cursor_incarnation_mismatch".to_string(),
5062 message: "spawn cursor belongs to a different daemon incarnation"
5063 .to_string(),
5064 detail: Some(serde_json::json!({
5065 "current_daemon_incarnation": current
5066 })),
5067 },
5068 )?])
5069 }
5070 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5071 &frame,
5072 ErrorBody {
5073 code: "spawn_cursor_too_old".to_string(),
5074 message: "spawn cursor predates the retained event ring".to_string(),
5075 detail: Some(serde_json::json!({
5076 "oldest_retained_cursor": oldest
5077 })),
5078 },
5079 )?]),
5080 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5081 0,
5082 frame.header.corr,
5083 format!("failed to open supervisor spawn subscription: {error}"),
5084 )),
5085 }
5086 }
5087
5088 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5089 let generation = self
5090 .registry
5091 .generation()
5092 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5093 let mut modules = Vec::new();
5094 for module in self.supervisor.list() {
5095 let status = module.status_for_control("list").map_err(|err| {
5096 RouterError::backend(
5097 0,
5098 frame.header.corr,
5099 format!("failed to read supervisor status: {err}"),
5100 )
5101 })?;
5102 let (configured, _) = module.configuration().map_err(|err| {
5103 RouterError::backend(
5104 0,
5105 frame.header.corr,
5106 format!("failed to read module configuration: {err}"),
5107 )
5108 })?;
5109 let image = module.running_image_agreement().await;
5111 let resources = Some(module.child_resource_usage());
5114 let mut verdict =
5115 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5116 verdict.spec = Some(launch_spec_agreement(
5117 &configured,
5118 status.process_alive,
5119 status.launch_spec_digest.as_ref(),
5120 ));
5121 let pending_reload = Some(verdict);
5122 modules.push(
5125 SupervisorEntry::new(
5126 status.module_id,
5127 status.state.to_string(),
5128 status.enabled,
5129 status.live,
5130 status.health.status,
5131 )
5132 .with_launch_nonce_env(Some(!cfg!(unix)))
5133 .with_launch_nonce_source(status.launch_nonce_source)
5134 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5135 .with_protocol(status.protocol)
5136 .with_pending_reload(pending_reload)
5137 .with_last_probe_ms(status.health.last_probe_ms)
5138 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5139 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5140 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5141 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5142 .with_restart_count(Some(status.restart_count))
5143 .with_max_restarts(Some(status.max_restarts))
5144 .with_lifetime_restarts(Some(status.lifetime_restarts))
5145 .with_spawn_generation(Some(status.spawn_generation))
5146 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5147 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5148 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5149 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5150 .with_resources(resources),
5151 );
5152 }
5153 let response = ClientControlResponse::SupervisorList {
5154 generation,
5155 modules,
5156 };
5157 Ok(vec![control_response_body_frame(
5158 &frame,
5159 &response,
5160 "ClientControlResponse::SupervisorList",
5161 )?])
5162 }
5163
5164 fn handle_supervisor_stderr_tail(
5165 &self,
5166 frame: Frame,
5167 module_id: String,
5168 max_lines: Option<u32>,
5169 max_bytes: Option<u32>,
5170 ) -> Result<Vec<Frame>, RouterError> {
5171 let Some(module) = self.supervisor.get(&module_id) else {
5172 return Ok(vec![control_error_frame(
5173 &frame,
5174 "unknown_module",
5175 format!("module_id '{module_id}' is not supervised"),
5176 )?]);
5177 };
5178
5179 let snapshot = module.stderr_tail(
5180 max_lines.map(|value| value as usize),
5181 max_bytes.map(|value| value as usize),
5182 );
5183
5184 let response = ClientControlResponse::SupervisorStderrTail {
5185 module_id,
5186 tail: StderrTail {
5187 capture: match snapshot.capture {
5188 CaptureState::Captured => StderrCaptureState::Captured,
5189 CaptureState::Incomplete { reason } => {
5190 StderrCaptureState::Incomplete { reason }
5191 }
5192 CaptureState::NotCaptured { reason } => {
5193 StderrCaptureState::NotCaptured { reason }
5194 }
5195 },
5196 entries: snapshot
5197 .entries
5198 .into_iter()
5199 .map(|entry| match entry {
5200 TailEntry::Line {
5201 text,
5202 truncated,
5203 at_ms,
5204 } => StderrTailEntry::Line {
5205 text,
5206 truncated,
5207 at_ms,
5208 },
5209 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5210 })
5211 .collect(),
5212 dropped_lines: snapshot.dropped_lines,
5213 },
5214 };
5215 Ok(vec![control_response_body_frame(
5216 &frame,
5217 &response,
5218 "ClientControlResponse::SupervisorStderrTail",
5219 )?])
5220 }
5221
5222 async fn handle_supervisor_terminals(
5223 &self,
5224 frame: Frame,
5225 module_id: String,
5226 ) -> Result<Vec<Frame>, RouterError> {
5227 let Some(module) = self.supervisor.get(&module_id) else {
5228 return Ok(vec![control_error_frame(
5229 &frame,
5230 "unknown_module",
5231 format!("module_id '{module_id}' is not supervised"),
5232 )?]);
5233 };
5234
5235 let terminals = module
5238 .read_durable_terminal_history()
5239 .await
5240 .map_err(|error| {
5241 RouterError::backend(
5242 0,
5243 frame.header.corr,
5244 format!("failed to read terminal history: {error}"),
5245 )
5246 })?;
5247 let response = ClientControlResponse::SupervisorTerminals {
5248 module_id,
5249 terminals,
5250 };
5251 Ok(vec![control_response_body_frame(
5252 &frame,
5253 &response,
5254 "ClientControlResponse::SupervisorTerminals",
5255 )?])
5256 }
5257
5258 fn handle_supervisor_routes(
5259 &self,
5260 frame: Frame,
5261 module_id: Option<String>,
5262 ) -> Result<Vec<Frame>, RouterError> {
5263 let modules = self
5264 .forwarding
5265 .route_census(module_id.as_deref())
5266 .map_err(RouterError::Forwarding)?
5267 .into_iter()
5268 .map(|(module_id, routes)| SupervisorRouteModule {
5269 module_id,
5270 routes: routes
5271 .into_iter()
5272 .map(|route| SupervisorRoute {
5273 consumer: match route.principal {
5274 Principal::Reserved { module_id } => {
5275 SupervisorRouteConsumer::Reserved { module_id }
5276 }
5277 Principal::Direct | Principal::Unverified => {
5278 SupervisorRouteConsumer::Direct {
5279 connection_id: route.goodbye_target.connection_id.get(),
5280 }
5281 }
5282 },
5283 age_ms: Instant::now()
5284 .saturating_duration_since(route.bound_at)
5285 .as_millis()
5286 .try_into()
5287 .unwrap_or(u64::MAX),
5288 draining: route.draining,
5289 drain_reason: route.drain_reason,
5290 })
5291 .collect(),
5292 })
5293 .collect();
5294 let response = ClientControlResponse::SupervisorRoutes { modules };
5295 Ok(vec![control_response_body_frame(
5296 &frame,
5297 &response,
5298 "ClientControlResponse::SupervisorRoutes",
5299 )?])
5300 }
5301
5302 async fn handle_supervisor_provenance(
5303 &self,
5304 frame: Frame,
5305 module_id: Option<String>,
5306 ) -> Result<Vec<Frame>, RouterError> {
5307 let mut selected = if let Some(module_id) = module_id {
5308 let Some(module) = self.supervisor.get(&module_id) else {
5309 return Ok(vec![control_error_frame(
5310 &frame,
5311 "unknown_module",
5312 format!("module_id '{module_id}' is not supervised"),
5313 )?]);
5314 };
5315 vec![module]
5316 } else {
5317 self.supervisor.list()
5318 };
5319
5320 let mut modules = Vec::with_capacity(selected.len());
5321 for module in selected.drain(..) {
5322 let (status, observed_image) = module
5323 .status_and_running_image_agreement()
5324 .await
5325 .map_err(|err| {
5326 RouterError::backend(
5327 0,
5328 frame.header.corr,
5329 format!("failed to read supervisor status: {err}"),
5330 )
5331 })?;
5332 let module_declared = self
5333 .registry
5334 .get_module(&status.module_id)
5335 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5336 .and_then(|registration| registration.manifest.provenance)
5337 .map(|build| ModuleDeclaredProvenance::Reported { build })
5338 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5339 #[cfg(test)]
5340 let running_image = match &self.provenance_probe_override {
5341 Some(result) => result.clone(),
5342 None => observed_image,
5343 };
5344 #[cfg(not(test))]
5345 let running_image = observed_image;
5346 modules.push(SupervisorModuleProvenance {
5347 module_id: status.module_id,
5348 module_declared,
5349 daemon_observed: SupervisorObservedProcess {
5350 launch_nonce_source: status.launch_nonce_source,
5351 pid: status.pid,
5352 spawned_at_ms: status.spawned_at_ms,
5353 spawned_from: status.spawned_from,
5354 running_image,
5355 },
5356 });
5357 }
5358 let daemon = SupervisorDaemonProvenance {
5359 daemon_build: self.daemon_provenance.build.clone(),
5360 daemon_observed: DaemonObservedProcess {
5361 pid: self.daemon_provenance.pid,
5362 started_at_ms: self
5363 .daemon_provenance
5364 .start_clock
5365 .map(|clock| clock.started_at_ms())
5366 .or(self.daemon_provenance.started_at_ms),
5367 running_image: self
5368 .daemon_provenance
5369 .probe
5370 .observe(
5371 self.daemon_provenance.pid,
5372 self.daemon_provenance.executable_path.as_deref(),
5373 self.daemon_provenance.executable_identity.clone(),
5374 self.daemon_provenance.process_start_time,
5375 )
5376 .await,
5377 },
5378 };
5379 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5380 Ok(vec![control_response_body_frame(
5381 &frame,
5382 &response,
5383 "ClientControlResponse::SupervisorProvenance",
5384 )?])
5385 }
5386
5387 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5388 self.refresh_capability_requirements();
5389 let generation = self
5390 .registry
5391 .generation()
5392 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5393 let modules = self
5394 .supervisor
5395 .list()
5396 .into_iter()
5397 .map(|module| {
5398 let status = module.status_for_control("health").map_err(|err| {
5399 RouterError::backend(
5400 0,
5401 frame.header.corr,
5402 format!("failed to read supervisor health: {err}"),
5403 )
5404 })?;
5405 let module_id = status.module_id;
5406 let capability_detail = self
5407 .capability_evaluator
5408 .required_problem_detail(&module_id);
5409 Ok(SupervisorHealthEntry {
5410 module_id,
5411 status: status.health.status,
5412 detail: append_capability_problem_detail(
5413 status.health.detail,
5414 capability_detail,
5415 ),
5416 metrics: status.health.metrics,
5417 consecutive_failures: status.health.consecutive_failures,
5418 late_answer_count: status.health.late_answer_count,
5419 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5420 last_action: status.health.last_action,
5421 last_action_ms: status.health.last_action_ms,
5422 last_probe_ms: status.health.last_probe_ms,
5423 })
5424 })
5425 .collect::<Result<Vec<_>, RouterError>>()?;
5426 let response = ClientControlResponse::SupervisorHealth {
5427 generation,
5428 modules,
5429 };
5430 Ok(vec![control_response_body_frame(
5431 &frame,
5432 &response,
5433 "ClientControlResponse::SupervisorHealth",
5434 )?])
5435 }
5436
5437 async fn handle_supervisor_restart(
5438 &self,
5439 frame: Frame,
5440 module_id: String,
5441 drain_timeout_ms: Option<u64>,
5442 requester: Principal,
5443 ) -> Result<Vec<Frame>, RouterError> {
5444 let operation_lock = self.supervisor.operation_lock();
5445 let _operation_guard = operation_lock.lock().await;
5446 let Some(module) = self.supervisor.get(&module_id) else {
5447 return Ok(vec![control_error_frame(
5448 &frame,
5449 "unknown_module",
5450 format!("module_id '{module_id}' is not supervised"),
5451 )?]);
5452 };
5453
5454 self.route_outages.mark_operator_action(&module_id);
5455 if let Err(err) = module
5456 .restart_requested(drain_timeout_ms, Some(requester))
5457 .await
5458 {
5459 self.route_outages
5460 .operator_action_ended_unrefused(&module_id);
5461 let (code, message) = match err {
5462 crate::supervise::SuperviseError::Disabled { .. } => {
5463 ("module_disabled", err.to_string())
5464 }
5465 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5466 ("swap_in_progress", err.to_string())
5467 }
5468 _ => (
5469 "target_unavailable",
5470 format!("failed to restart module_id '{module_id}': {err}"),
5471 ),
5472 };
5473 return Ok(vec![control_error_frame(&frame, code, message)?]);
5474 }
5475
5476 let response = ClientControlResponse::SupervisorAck {
5477 module_id,
5478 applied: true,
5479 };
5480 Ok(vec![control_response_body_frame(
5481 &frame,
5482 &response,
5483 "ClientControlResponse::SupervisorAck",
5484 )?])
5485 }
5486
5487 async fn handle_supervisor_swap(
5491 &self,
5492 frame: Frame,
5493 module_id: String,
5494 ready_timeout_ms: Option<u64>,
5495 requester: Principal,
5496 ) -> Result<Vec<Frame>, RouterError> {
5497 let module = {
5504 let operation_lock = self.supervisor.operation_lock();
5505 let _operation_guard = operation_lock.lock().await;
5506 self.supervisor.get(&module_id)
5507 };
5508 let Some(module) = module else {
5509 return Ok(vec![control_error_frame(
5510 &frame,
5511 "unknown_module",
5512 format!("module_id '{module_id}' is not supervised"),
5513 )?]);
5514 };
5515
5516 self.route_outages.mark_operator_action(&module_id);
5517 if let Err(err) = module
5518 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5519 .await
5520 {
5521 self.route_outages
5522 .operator_action_ended_unrefused(&module_id);
5523 use crate::supervise::SuperviseError;
5524 let message = err.to_string();
5525 let error = match err {
5526 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5527 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5528 code: "swap_refused".to_string(),
5529 message,
5530 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5531 },
5532 SuperviseError::SwapFailed {
5533 arm,
5534 candidate_exit,
5535 ..
5536 } => ErrorBody {
5537 code: "swap_failed".to_string(),
5538 message,
5539 detail: Some(serde_json::json!({
5540 "arm": arm.as_str(),
5541 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5542 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5543 })),
5544 },
5545 _ => ErrorBody::new(
5546 "target_unavailable",
5547 format!("failed to swap module_id '{module_id}': {message}"),
5548 ),
5549 };
5550 return Ok(vec![control_error_body_frame(&frame, error)?]);
5551 }
5552 self.route_outages
5556 .operator_action_ended_unrefused(&module_id);
5557
5558 let response = ClientControlResponse::SupervisorAck {
5559 module_id,
5560 applied: true,
5561 };
5562 Ok(vec![control_response_body_frame(
5563 &frame,
5564 &response,
5565 "ClientControlResponse::SupervisorAck",
5566 )?])
5567 }
5568
5569 async fn handle_supervisor_reload(
5570 &self,
5571 frame: Frame,
5572 module_id: String,
5573 requester: Principal,
5574 ) -> Result<Vec<Frame>, RouterError> {
5575 let operation_lock = self.supervisor.operation_lock();
5576 let _operation_guard = operation_lock.lock().await;
5577 let Some(module) = self.supervisor.get(&module_id) else {
5578 return Ok(vec![control_error_frame(
5579 &frame,
5580 "unknown_module",
5581 format!("module_id '{module_id}' is not supervised"),
5582 )?]);
5583 };
5584
5585 self.route_outages.mark_operator_action(&module_id);
5586 if let Err(err) = module.reload_requested(Some(requester)).await {
5587 self.route_outages
5588 .operator_action_ended_unrefused(&module_id);
5589 let (code, message) = match err {
5590 crate::supervise::SuperviseError::Disabled { .. } => {
5591 ("module_disabled", err.to_string())
5592 }
5593 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5594 ("swap_in_progress", err.to_string())
5595 }
5596 _ => (
5597 "reload_failed",
5598 format!("failed to reload module_id '{module_id}': {err}"),
5599 ),
5600 };
5601 return Ok(vec![control_error_frame(&frame, code, message)?]);
5602 }
5603
5604 let response = ClientControlResponse::SupervisorAck {
5605 module_id,
5606 applied: true,
5607 };
5608 Ok(vec![control_response_body_frame(
5609 &frame,
5610 &response,
5611 "ClientControlResponse::SupervisorAck",
5612 )?])
5613 }
5614
5615 async fn handle_supervisor_rescan(
5616 &self,
5617 frame: Frame,
5618 preview: bool,
5619 requester: Principal,
5620 ) -> Result<Vec<Frame>, RouterError> {
5621 let Some(context) = self.rescan.clone() else {
5622 return Ok(vec![control_error_frame(
5623 &frame,
5624 "rescan_unavailable",
5625 "the daemon was not started with a reloadable config path".to_string(),
5626 )?]);
5627 };
5628
5629 let operation_lock = self.supervisor.operation_lock();
5630 let _operation_guard = operation_lock.lock().await;
5631 let loaded = match crate::daemon_config::load_with_activity(&context.config_path) {
5632 Ok(config) => config,
5633 Err(err) => {
5634 return Ok(vec![control_error_frame(
5635 &frame,
5636 "invalid_daemon_config",
5637 format!("supervisor rescan rejected daemon config: {err}"),
5638 )?])
5639 }
5640 };
5641 let Some((config, activity_config)) = loaded else {
5649 return Ok(vec![control_error_frame(
5650 &frame,
5651 "invalid_daemon_config",
5652 format!(
5653 "daemon config not found at {}; refusing to rescan (an absent config would \
5654 retire every supervised module)",
5655 context.config_path.display()
5656 ),
5657 )?]);
5658 };
5659 let (
5660 configured_port,
5661 storage_config,
5662 admission_facts_carrier_module_id,
5663 admission_facts_targets,
5664 scope_authority_owners,
5665 privileged_operator_prompt,
5666 modules,
5667 reserved_capabilities,
5668 ) = (
5669 config.port,
5670 config.storage,
5671 config.admission_facts_carrier_module_id,
5672 config.admission_facts_targets,
5673 config.scope_authority_owners,
5674 config.privileged_operator_prompt,
5675 config.modules,
5676 config.reserved_capabilities,
5677 );
5678
5679 let mut restart_required = Vec::new();
5688 for section in RestartRequiredSection::ALL {
5689 let changed = match section {
5690 RestartRequiredSection::Port => configured_port != context.configured_port,
5691 RestartRequiredSection::Storage => storage_config != context.storage_config,
5692 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5693 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5694 }
5695 RestartRequiredSection::AdmissionFactsTargets => {
5696 admission_facts_targets != context.admission_facts_targets
5697 }
5698 RestartRequiredSection::ScopeAuthorityOwners => {
5699 scope_authority_owners != context.scope_authority_owners
5700 }
5701 RestartRequiredSection::PrivilegedOperatorPrompt => {
5702 privileged_operator_prompt != context.privileged_operator_prompt
5703 }
5704 };
5705 if changed {
5706 restart_required.push(section.label().to_string());
5707 }
5708 }
5709 if !restart_required.is_empty() {
5710 warn!(
5711 config_path = %context.config_path.display(),
5712 sections = %restart_required.join(", "),
5713 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5714 );
5715 }
5716
5717 for configured in &modules {
5718 if let Err(err) = validate_spec(&configured.module_spec()) {
5719 return Ok(vec![control_error_frame(
5720 &frame,
5721 "invalid_daemon_config",
5722 format!("supervisor rescan rejected daemon config: {err}"),
5723 )?]);
5724 }
5725 }
5726
5727 let configured_capabilities = modules
5728 .iter()
5729 .map(|module| (module.module_id.clone(), module.enabled))
5730 .collect::<Vec<_>>();
5731 let preview_capability_warnings = if preview {
5732 let (_, registrations) = self.runtime_capability_snapshot()?;
5733 let current_modules = self
5734 .supervisor
5735 .list()
5736 .into_iter()
5737 .map(|module| module.module_id().to_string())
5738 .collect::<BTreeSet<_>>();
5739 let resulting_modules = configured_capabilities.clone();
5740 let removed = current_modules
5741 .into_iter()
5742 .filter(|module_id| {
5743 !resulting_modules
5744 .iter()
5745 .any(|(configured_id, _)| configured_id == module_id)
5746 })
5747 .collect::<Vec<_>>();
5748 self.capability_evaluator.preview_removal_warnings(
5749 resulting_modules,
5750 &removed,
5751 ®istrations,
5752 )
5753 } else {
5754 Vec::new()
5755 };
5756 let result = match self
5757 .reconcile_supervised_modules(&context.supervisor, modules, preview, requester.clone())
5758 .await
5759 {
5760 Ok(result) => result,
5761 Err(message) => {
5762 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5763 }
5764 };
5765 if !preview {
5766 if let Some(ledger) = &self.activity {
5767 ledger.module_rescanned(&result.added, &result.removed, requester);
5768 }
5769 if let Err(code) = self.configure_activity(activity_config).await {
5770 warn!(
5771 code,
5772 "activity retention could not be applied during rescan"
5773 );
5774 }
5775 self.capability_evaluator
5776 .configure(configured_capabilities, reserved_capabilities);
5777 self.capability_evaluator.wake_deadline_loop();
5778 self.refresh_capability_requirements();
5779 }
5780 let mut result = result;
5781 result.restart_required = restart_required;
5782 result.capability_warnings = preview_capability_warnings;
5783 let response = ClientControlResponse::SupervisorRescan { result };
5784 Ok(vec![control_response_body_frame(
5785 &frame,
5786 &response,
5787 "ClientControlResponse::SupervisorRescan",
5788 )?])
5789 }
5790
5791 async fn handle_supervisor_release_reserved(
5792 &self,
5793 frame: Frame,
5794 module_id: String,
5795 ) -> Result<Vec<Frame>, RouterError> {
5796 let Some(context) = self.rescan.clone() else {
5797 return Ok(vec![control_error_frame(
5798 &frame,
5799 "release_unavailable",
5800 "reserved-id release requires a daemon started with a reloadable config path",
5801 )?]);
5802 };
5803 let operation_lock = self.supervisor.operation_lock();
5804 let _operation_guard = operation_lock.lock().await;
5805 let loaded = match crate::daemon_config::load(&context.config_path) {
5806 Ok(Some(config)) => config,
5807 Ok(None) => {
5808 return Ok(vec![control_error_frame(
5809 &frame,
5810 "invalid_daemon_config",
5811 format!(
5812 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5813 context.config_path.display()
5814 ),
5815 )?])
5816 }
5817 Err(err) => {
5818 return Ok(vec![control_error_frame(
5819 &frame,
5820 "invalid_daemon_config",
5821 format!("unable to verify reserved-id release against daemon config: {err}"),
5822 )?])
5823 }
5824 };
5825 if loaded
5826 .modules
5827 .iter()
5828 .any(|configured| configured.module_id == module_id)
5829 {
5830 return Ok(vec![control_error_frame(
5831 &frame,
5832 "reserved_module_configured",
5833 format!(
5834 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5835 ),
5836 )?]);
5837 }
5838 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5839 return Ok(vec![control_error_frame(
5840 &frame,
5841 "reserved_gate_not_retained",
5842 format!(
5843 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5844 ),
5845 )?]);
5846 }
5847
5848 let response = ClientControlResponse::SupervisorAck {
5849 module_id,
5850 applied: true,
5851 };
5852 Ok(vec![control_response_body_frame(
5853 &frame,
5854 &response,
5855 "ClientControlResponse::SupervisorAck",
5856 )?])
5857 }
5858
5859 async fn reconcile_supervised_modules(
5868 &self,
5869 supervisor: &Supervisor,
5870 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5871 preview: bool,
5872 requester: Principal,
5873 ) -> Result<SupervisorRescanResult, String> {
5874 let mut current = BTreeMap::new();
5875 for module in self.supervisor.list() {
5876 let (spec, health) = module.configuration().map_err(|err| {
5877 format!(
5878 "failed to read configuration for module_id '{}': {err}",
5879 module.module_id()
5880 )
5881 })?;
5882 let enabled = module
5883 .status()
5884 .map_err(|err| {
5885 format!(
5886 "failed to read status for module_id '{}': {err}",
5887 module.module_id()
5888 )
5889 })?
5890 .enabled;
5891 current.insert(
5892 module.module_id().to_string(),
5893 (module, spec, health, enabled),
5894 );
5895 }
5896 let configured = configured_modules
5897 .into_iter()
5898 .map(|module| (module.module_id.clone(), module))
5899 .collect::<BTreeMap<_, _>>();
5900
5901 let added = configured
5902 .keys()
5903 .filter(|module_id| !current.contains_key(*module_id))
5904 .cloned()
5905 .collect::<Vec<_>>();
5906 let removed = current
5907 .keys()
5908 .filter(|module_id| !configured.contains_key(*module_id))
5909 .cloned()
5910 .collect::<Vec<_>>();
5911 let mut changed_pending_reload = Vec::new();
5912 let mut configuration_changes = BTreeSet::new();
5913 let mut enabled_changes = BTreeSet::new();
5914 let mut unchanged = 0_u32;
5915
5916 for (module_id, configured_module) in &configured {
5917 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5918 else {
5919 continue;
5920 };
5921 let launch_changed = *current_spec != configured_module.module_spec();
5925 let configuration_changed =
5926 launch_changed || *current_health != configured_module.health;
5927 let enabled_changed = *current_enabled != configured_module.enabled;
5928 if configuration_changed {
5929 configuration_changes.insert(module_id.clone());
5930 }
5931 if launch_changed {
5932 changed_pending_reload.push(module_id.clone());
5933 }
5934 if enabled_changed {
5935 enabled_changes.insert(module_id.clone());
5936 }
5937 if !configuration_changed && !enabled_changed {
5938 unchanged = unchanged.saturating_add(1);
5939 }
5940 }
5941
5942 if preview {
5948 return Ok(SupervisorRescanResult {
5949 added,
5950 removed,
5951 changed_pending_reload,
5952 enabled_changes: enabled_changes.iter().cloned().collect(),
5953 unchanged,
5954 preview: true,
5955 restart_required: Vec::new(),
5959 capability_warnings: Vec::new(),
5960 });
5961 }
5962
5963 for module_id in &removed {
5964 let module = ¤t
5965 .get(module_id)
5966 .expect("removed module came from current supervisor state")
5967 .0;
5968 module
5969 .retire_requested(Some(requester.clone()))
5970 .await
5971 .map_err(|err| {
5972 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5973 })?;
5974 self.supervisor.record_rescan_removal(module_id);
6008 self.supervisor.retire(module_id);
6009 self.route_outages.forget(module_id);
6010 }
6011
6012 for module_id in configured.keys() {
6013 let Some((module, _, _, _)) = current.get(module_id) else {
6014 continue;
6015 };
6016 let configured_module = configured
6017 .get(module_id)
6018 .expect("configured module id came from configured map");
6019 if configuration_changes.contains(module_id) {
6020 module
6021 .update_configuration(
6022 configured_module.module_spec(),
6023 configured_module.health.clone(),
6024 configured_module.drain_timeout_ms,
6025 )
6026 .await
6027 .map_err(|err| {
6028 format!(
6029 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6030 )
6031 })?;
6032 }
6033 if enabled_changes.contains(module_id) {
6034 self.route_outages.mark_operator_action(module_id);
6037 module
6038 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6039 .await
6040 .map_err(|err| {
6041 self.route_outages.operator_action_ended_unrefused(module_id);
6042 format!(
6043 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6044 configured_module.enabled
6045 )
6046 })?;
6047 }
6048 }
6049
6050 for module_id in &added {
6051 let configured_module = configured
6052 .get(module_id)
6053 .expect("added module id came from configured map");
6054 supervisor
6055 .supervise_configured_requested(
6056 configured_module.module_spec(),
6057 configured_module.enabled,
6058 configured_module.health.clone(),
6059 configured_module.drain_timeout_ms,
6060 configured_module.restart,
6061 Some(requester.clone()),
6062 )
6063 .map_err(|err| {
6064 format!("failed to add module_id '{module_id}' during rescan: {err}")
6065 })?;
6066 }
6067
6068 Ok(SupervisorRescanResult {
6069 added,
6070 removed,
6071 changed_pending_reload,
6072 enabled_changes: enabled_changes.iter().cloned().collect(),
6073 unchanged,
6074 preview: false,
6075 restart_required: Vec::new(),
6078 capability_warnings: Vec::new(),
6079 })
6080 }
6081
6082 async fn handle_supervisor_set_enabled(
6083 &self,
6084 frame: Frame,
6085 module_id: String,
6086 enabled: bool,
6087 requester: Principal,
6088 ) -> Result<Vec<Frame>, RouterError> {
6089 let operation_lock = self.supervisor.operation_lock();
6090 let _operation_guard = operation_lock.lock().await;
6091 let Some(module) = self.supervisor.get(&module_id) else {
6092 return Ok(vec![control_error_frame(
6093 &frame,
6094 "unknown_module",
6095 format!("module_id '{module_id}' is not supervised"),
6096 )?]);
6097 };
6098
6099 self.route_outages.mark_operator_action(&module_id);
6102 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6103 Ok(applied) => applied,
6104 Err(err) => {
6105 self.route_outages
6106 .operator_action_ended_unrefused(&module_id);
6107 return Ok(vec![control_error_frame(
6108 &frame,
6109 "target_unavailable",
6110 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6111 )?]);
6112 }
6113 };
6114 if !applied {
6115 self.route_outages
6118 .operator_action_ended_unrefused(&module_id);
6119 }
6120
6121 self.capability_evaluator.wake_deadline_loop();
6122 self.refresh_capability_requirements();
6123 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6124 Ok(vec![control_response_body_frame(
6125 &frame,
6126 &response,
6127 "ClientControlResponse::SupervisorAck",
6128 )?])
6129 }
6130
6131 async fn handle_supervisor_health_probe(
6132 &self,
6133 frame: Frame,
6134 module_id: String,
6135 ) -> Result<Vec<Frame>, RouterError> {
6136 self.refresh_capability_requirements();
6137 let Some(registration) = self
6138 .registry
6139 .get_module(&module_id)
6140 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6141 else {
6142 return Ok(vec![control_error_frame(
6143 &frame,
6144 "unknown_module",
6145 format!("module_id '{module_id}' is not registered"),
6146 )?]);
6147 };
6148
6149 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6162 {
6163 return Ok(vec![control_error_frame(
6164 &frame,
6165 "health_not_advertised",
6166 format!("module_id '{module_id}' did not advertise health.check"),
6167 )?]);
6168 }
6169
6170 let deadline = Instant::now() + self.health_probe_timeout;
6171 let pending = match self.forwarding.begin_module_control_rpc_for(
6172 &module_id,
6173 MODULE_CONTROL_OP_HEALTH_CHECK,
6174 deadline,
6175 ) {
6176 Ok(pending) => pending,
6177 Err(err) => {
6178 return Ok(vec![control_error_frame(
6179 &frame,
6180 forwarding_error_code(&err),
6181 err.to_string(),
6182 )?])
6183 }
6184 };
6185
6186 let PendingModuleControlRpc {
6187 endpoint,
6188 module_sink,
6189 negotiated_ver,
6190 corr: probe_corr,
6191 receiver,
6192 } = pending;
6193 let mut guard =
6194 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6195 let probe_body =
6196 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6197 RouterError::backend(
6198 0,
6199 frame.header.corr,
6200 format!("failed to encode health.check request: {err}"),
6201 )
6202 })?;
6203 let probe_frame = Frame::build_with_version(
6204 negotiated_ver,
6205 FrameType::Request,
6206 control_flags(),
6207 0,
6208 0,
6209 probe_corr,
6210 probe_body,
6211 )
6212 .map_err(RouterError::FrameBuild)?;
6213
6214 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6217 Ok(Ok(())) => {}
6218 Ok(Err(err)) => {
6219 return Ok(vec![control_error_frame(
6220 &frame,
6221 "target_unavailable",
6222 err.to_string(),
6223 )?]);
6224 }
6225 Err(_) => {
6226 return Ok(vec![control_error_frame(
6227 &frame,
6228 "module_timeout",
6229 format!(
6230 "module_id '{module_id}' did not answer health.check within {:?}",
6231 self.health_probe_timeout
6232 ),
6233 )?]);
6234 }
6235 }
6236
6237 match timeout_at(deadline, receiver).await {
6238 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6239 guard.disarm();
6240 let Some(report) = response.health_report() else {
6241 return Ok(vec![control_error_frame(
6242 &frame,
6243 "invalid_control_body",
6244 "health.check RPC returned a non-health response",
6245 )?]);
6246 };
6247 let HealthReport {
6252 status,
6253 detail,
6254 metrics,
6255 } = report;
6256 let capability_detail = self
6257 .capability_evaluator
6258 .required_problem_detail(&module_id);
6259 let response = ClientControlResponse::SupervisorHealthProbe {
6260 module_id,
6261 status,
6262 detail: append_capability_problem_detail(detail, capability_detail),
6263 metrics,
6264 };
6265 Ok(vec![control_response_body_frame(
6266 &frame,
6267 &response,
6268 "ClientControlResponse::SupervisorHealthProbe",
6269 )?])
6270 }
6271 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6272 guard.disarm();
6273 Ok(vec![control_error_body_frame(&frame, body)?])
6274 }
6275 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6276 guard.disarm();
6277 Ok(vec![control_error_frame(
6278 &frame,
6279 "target_unavailable",
6280 message,
6281 )?])
6282 }
6283 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6284 guard.disarm();
6285 Ok(vec![control_error_frame(
6286 &frame,
6287 "invalid_control_body",
6288 message,
6289 )?])
6290 }
6291 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6292 guard.disarm();
6293 Ok(vec![control_error_frame(
6294 &frame,
6295 "invalid_control_body",
6296 format!("expected module-control op '{expected}', got '{actual}'"),
6297 )?])
6298 }
6299 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6300 guard.disarm();
6301 Ok(vec![control_error_frame(
6302 &frame,
6303 "module_timeout",
6304 format!(
6305 "module_id '{module_id}' answered health.check after {:?}",
6306 self.health_probe_timeout
6307 ),
6308 )?])
6309 }
6310 Ok(Err(_)) => Ok(vec![control_error_frame(
6311 &frame,
6312 "target_unavailable",
6313 "health.check waiter was canceled before the module responded",
6314 )?]),
6315 Err(_) => Ok(vec![control_error_frame(
6316 &frame,
6317 "module_timeout",
6318 format!(
6319 "module_id '{module_id}' did not answer health.check within {:?}",
6320 self.health_probe_timeout
6321 ),
6322 )?]),
6323 }
6324 }
6325
6326 fn supervisor_status(
6327 &self,
6328 module_id: &str,
6329 corr: u64,
6330 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6331 self.supervisor
6332 .get(module_id)
6333 .map(|module| {
6334 let warming = module.is_warming_for_control("status").map_err(|err| {
6335 RouterError::backend(
6336 0,
6337 corr,
6338 format!(
6339 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6340 ),
6341 )
6342 })?;
6343 module.status_for_control("status").map_err(|err| {
6344 RouterError::backend(
6345 0,
6346 corr,
6347 format!(
6348 "failed to read supervisor status for module_id '{module_id}': {err}"
6349 ),
6350 )
6351 }).map(|status| (status, warming))
6352 })
6353 .transpose()
6354 }
6355
6356 fn guard_module_control_op(
6357 &self,
6358 frame: &Frame,
6359 module_id: &str,
6360 op: &str,
6361 ) -> Result<Option<Frame>, RouterError> {
6362 if self.module_grants_op(module_id, op, frame.header.corr)? {
6363 return Ok(None);
6364 }
6365
6366 Ok(Some(control_error_frame(
6367 frame,
6368 "op_not_allowed",
6369 format!("module_id '{module_id}' did not grant control op '{op}'"),
6370 )?))
6371 }
6372
6373 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6374 let Some(registration) = self
6375 .registry
6376 .get_module(module_id)
6377 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6378 else {
6379 return Ok(false);
6380 };
6381 Ok(module_registration_grants_op(®istration.control_ops, op))
6382 }
6383
6384 fn handle_status_update(
6385 &self,
6386 endpoint: ModuleEndpointId,
6387 frame: Frame,
6388 ) -> Result<Vec<Frame>, RouterError> {
6389 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6390 Ok(update) => update,
6391 Err(err) => {
6392 if is_known_module_push_op(&frame.body) {
6397 return Ok(vec![control_error_frame(
6398 &frame,
6399 "invalid_control_body",
6400 format!("malformed module control push body: {err}"),
6401 )?]);
6402 }
6403 return Ok(Vec::new());
6404 }
6405 };
6406
6407 match update {
6408 ModuleControlPush::RouteStatus {
6409 route_channel,
6410 route_epoch,
6411 status,
6412 } => {
6413 self.forwarding
6414 .cache_status(endpoint, route_channel, route_epoch, status)
6415 .map_err(RouterError::Forwarding)?;
6416 }
6417 }
6418 Ok(Vec::new())
6419 }
6420
6421 fn handle_route_poll(
6422 &self,
6423 ctx: &RouteCtx,
6424 frame: Frame,
6425 route_channel: u16,
6426 route_epoch: u32,
6427 kind: PollKind,
6428 ) -> Result<Vec<Frame>, RouterError> {
6429 let snapshot = self
6430 .forwarding
6431 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6432 .map_err(RouterError::Forwarding)?;
6433 let response = match (kind, snapshot) {
6434 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6435 ClientControlResponse::RoutePoll {
6436 route_channel,
6437 route_epoch,
6438 status,
6439 live: None,
6440 }
6441 }
6442 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6443 route_channel,
6444 route_epoch,
6445 status: None,
6446 live: None,
6447 },
6448 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6449 let live = self
6475 .process_liveness
6476 .as_ref()
6477 .and_then(|source| source.process_live(&module_id))
6478 .unwrap_or(true);
6479 ClientControlResponse::RoutePoll {
6480 route_channel,
6481 route_epoch,
6482 status: None,
6483 live: Some(live),
6484 }
6485 }
6486 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6487 route_channel,
6488 route_epoch,
6489 status: None,
6490 live: Some(false),
6491 },
6492 };
6493
6494 Ok(vec![control_response_body_frame(
6495 &frame,
6496 &response,
6497 "ClientControlResponse::RoutePoll",
6498 )?])
6499 }
6500
6501 pub(crate) fn observe_module_control_completion(
6502 &self,
6503 completion: ModuleControlRpcCompletion,
6504 ) -> bool {
6505 match completion {
6506 ModuleControlRpcCompletion::Unknown => false,
6507 ModuleControlRpcCompletion::Settled => true,
6508 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6509 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6510 info!(
6511 module_id = %module_id,
6512 latency_ms,
6513 "late health.check answer proves the module is alive"
6514 );
6515 match self
6516 .supervisor
6517 .record_late_health_answer(&module_id, latency_ms)
6518 {
6519 Ok(true) => {}
6520 Ok(false) => debug!(
6521 module_id = %module_id,
6522 latency_ms,
6523 "late health.check answer has no active supervisor snapshot"
6524 ),
6525 Err(err) => warn!(
6526 module_id = %module_id,
6527 latency_ms,
6528 error = %err,
6529 "failed to record late health.check answer"
6530 ),
6531 }
6532 true
6533 }
6534 }
6535 }
6536
6537 fn refuse_to_end_module_connection_for_a_client(
6558 &self,
6559 module_connection_id: ConnectionId,
6560 corr: u64,
6561 err: ForwardingError,
6562 ) -> Result<(), RouterError> {
6563 if let ForwardingError::ConnectionClosing { connection_id } = err {
6564 if connection_id != module_connection_id {
6565 warn!(
6566 module_connection_id = module_connection_id.get(),
6567 client_connection_id = connection_id.get(),
6568 corr,
6569 "dropping a route.bind response for a closing client; the module connection keeps serving"
6570 );
6571 return Ok(());
6572 }
6573 }
6574 Err(RouterError::Forwarding(err))
6575 }
6576
6577 fn handle_module_relay_response(
6578 &self,
6579 connection_id: ConnectionId,
6580 frame: Frame,
6581 ) -> Result<Vec<Frame>, RouterError> {
6582 let mut secondary_error = None;
6583 let outcome = match frame.header.ty {
6584 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6585 Ok(probe) if probe.op == "route.bind" => {
6586 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6587 Ok(ModuleControlResponse::RouteBindAck {}) => {
6588 RouteBindRelayOutcome::Accepted
6589 }
6590 Ok(other) => {
6591 let message =
6592 format!("route.bind response carried unexpected body: {other:?}");
6593 secondary_error = Some(control_error_frame(
6594 &frame,
6595 "invalid_control_body",
6596 message.clone(),
6597 )?);
6598 RouteBindRelayOutcome::ModuleGone(message)
6599 }
6600 Err(err) => {
6601 let message = format!("malformed route.bind response body: {err}");
6602 secondary_error = Some(control_error_frame(
6603 &frame,
6604 "invalid_control_body",
6605 message.clone(),
6606 )?);
6607 RouteBindRelayOutcome::ModuleGone(message)
6608 }
6609 }
6610 }
6611 Ok(probe) => {
6612 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6613 {
6614 Ok(response) => ModuleControlRpcOutcome::Response(response),
6615 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6616 "malformed {} response body: {err}",
6617 probe.op
6618 )),
6619 };
6620 let completion = self
6621 .forwarding
6622 .complete_module_control_rpc(
6623 connection_id,
6624 frame.header.corr,
6625 Some(&probe.op),
6626 outcome,
6627 )
6628 .map_err(RouterError::Forwarding)?;
6629 if !self.observe_module_control_completion(completion) {
6630 debug!(
6631 connection_id = connection_id.get(),
6632 corr = frame.header.corr,
6633 op = %probe.op,
6634 "dropping late or unknown module-control RPC response"
6635 );
6636 }
6637 return Ok(Vec::new());
6638 }
6639 Err(err) => {
6640 if let Some(expected_op) = self
6641 .forwarding
6642 .pending_module_control_op(connection_id, frame.header.corr)
6643 .map_err(RouterError::Forwarding)?
6644 {
6645 let completion = self
6646 .forwarding
6647 .complete_module_control_rpc(
6648 connection_id,
6649 frame.header.corr,
6650 None,
6651 ModuleControlRpcOutcome::MalformedResponse(format!(
6652 "malformed {expected_op} response body: {err}"
6653 )),
6654 )
6655 .map_err(RouterError::Forwarding)?;
6656 if !self.observe_module_control_completion(completion) {
6657 debug!(
6658 connection_id = connection_id.get(),
6659 corr = frame.header.corr,
6660 "dropping late malformed module-control RPC response"
6661 );
6662 }
6663 return Ok(Vec::new());
6664 }
6665 let message = format!("malformed route.bind response body: {err}");
6666 secondary_error = Some(control_error_frame(
6667 &frame,
6668 "invalid_control_body",
6669 message.clone(),
6670 )?);
6671 RouteBindRelayOutcome::ModuleGone(message)
6672 }
6673 },
6674 FrameType::Error => {
6675 if self
6676 .forwarding
6677 .pending_module_control_op(connection_id, frame.header.corr)
6678 .map_err(RouterError::Forwarding)?
6679 .is_some()
6680 {
6681 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6682 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6683 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6684 "malformed module-control ERROR body: {err}"
6685 )),
6686 };
6687 let completion = self
6688 .forwarding
6689 .complete_module_control_rpc(
6690 connection_id,
6691 frame.header.corr,
6692 None,
6693 outcome,
6694 )
6695 .map_err(RouterError::Forwarding)?;
6696 if !self.observe_module_control_completion(completion) {
6697 debug!(
6698 connection_id = connection_id.get(),
6699 corr = frame.header.corr,
6700 "dropping late or unknown module-control RPC error"
6701 );
6702 }
6703 return Ok(Vec::new());
6704 }
6705 match serde_json::from_slice::<ErrorBody>(&frame.body) {
6706 Ok(body) => RouteBindRelayOutcome::Rejected(body),
6707 Err(err) => {
6708 let message = format!("malformed route.bind ERROR body: {err}");
6709 secondary_error = Some(control_error_frame(
6710 &frame,
6711 "invalid_control_body",
6712 message.clone(),
6713 )?);
6714 RouteBindRelayOutcome::ModuleGone(message)
6715 }
6716 }
6717 }
6718 ty => {
6719 return Ok(vec![control_error_frame(
6720 &frame,
6721 "unsupported_control_frame",
6722 format!("unsupported module channel-0 frame {ty:?}"),
6723 )?])
6724 }
6725 };
6726
6727 let settled =
6728 self.forwarding
6729 .complete_pending_relay(connection_id, frame.header.corr, outcome);
6730 let completion = match settled {
6731 Ok(completion) => completion,
6732 Err(err) => {
6733 self.refuse_to_end_module_connection_for_a_client(
6734 connection_id,
6735 frame.header.corr,
6736 err,
6737 )?;
6738 return Ok(secondary_error.into_iter().collect());
6739 }
6740 };
6741 if let Some(target) = completion.abandoned.as_ref() {
6742 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6743 }
6744 if !completion.settled {
6745 debug!(
6746 connection_id = connection_id.get(),
6747 corr = frame.header.corr,
6748 frame_type = ?frame.header.ty,
6749 "dropping late or unknown route.bind relay response"
6750 );
6751 }
6752 Ok(secondary_error.into_iter().collect())
6753 }
6754
6755 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6756 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6757 self.cleanup_connection_with_end_reason(
6761 connection_id,
6762 RegistrationEndReason::ExplicitGoodbye,
6763 )
6764 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6765 Ok(Vec::new())
6766 }
6767}
6768
6769impl Default for ControlHandler {
6770 fn default() -> Self {
6771 Self::new(Arc::new(Registry::default()))
6772 }
6773}
6774
6775impl crate::supervise::SwapPromotionObserver for ControlHandler {
6776 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6777 self.apply_registration_capabilities(registration);
6778 }
6779}
6780
6781fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6782 CapabilityRequirementStatus {
6783 consumer: status.consumer,
6784 capability: status.capability,
6785 need: match status.need {
6786 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6787 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6788 },
6789 verdict: status.verdict.as_str().to_string(),
6790 episode_seq: status.episode_seq,
6791 config_satisfiable: status.config_satisfiable,
6792 runtime_available: status.runtime_available,
6793 detail: status.detail,
6794 }
6795}
6796
6797fn append_capability_problem_detail(
6798 detail: Option<String>,
6799 capability_detail: Option<String>,
6800) -> Option<String> {
6801 match (detail, capability_detail) {
6802 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6803 (Some(detail), None) => Some(detail),
6804 (None, Some(capability_detail)) => Some(capability_detail),
6805 (None, None) => None,
6806 }
6807}
6808
6809fn subc_ops() -> Vec<String> {
6810 SUBC_CONTROL_OPS
6811 .iter()
6812 .map(|op| (*op).to_string())
6813 .collect()
6814}
6815
6816fn module_subc_ops() -> Vec<String> {
6817 SUBC_CONTROL_OPS
6818 .iter()
6819 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6820 .map(|op| (*op).to_string())
6821 .collect()
6822}
6823
6824#[cfg(test)]
6825fn module_baseline_control_ops() -> Vec<String> {
6826 MODULE_BASELINE_CONTROL_OPS
6827 .iter()
6828 .map(|op| (*op).to_string())
6829 .collect()
6830}
6831
6832fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6833 let mut seen = HashSet::new();
6834 let mut effective = Vec::new();
6835 for op in MODULE_BASELINE_CONTROL_OPS {
6836 if seen.insert((*op).to_string()) {
6837 effective.push((*op).to_string());
6838 }
6839 }
6840 for op in declared.unwrap_or_default() {
6841 if seen.insert(op.clone()) {
6842 effective.push(op);
6843 }
6844 }
6845 effective
6846}
6847
6848fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6849 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6850}
6851
6852fn target_module_id(target: &RouteTarget) -> &str {
6853 match target {
6854 RouteTarget::ToolProvider { module_id }
6855 | RouteTarget::ManagementSurface { module_id }
6856 | RouteTarget::InternalService { module_id, .. } => module_id,
6857 }
6858}
6859
6860fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6861 roles.iter().any(|role| match (target, role) {
6862 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6863 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6864 (
6865 RouteTarget::InternalService { service_id, .. },
6866 ProviderRole::InternalService {
6867 service_id: provided,
6868 ..
6869 },
6870 ) => service_id == provided,
6871 _ => false,
6872 })
6873}
6874
6875fn is_routable_role(role: &ProviderRole) -> bool {
6876 matches!(
6877 role,
6878 ProviderRole::ToolProvider { .. }
6879 | ProviderRole::ManagementSurface { .. }
6880 | ProviderRole::InternalService { .. }
6881 )
6882}
6883
6884#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6885enum ControlRequestBodyError {
6886 UnknownOp,
6887 InvalidBody,
6888}
6889
6890#[derive(Debug, Deserialize)]
6891struct ControlOpProbe {
6892 op: String,
6893}
6894
6895const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6898
6899fn is_known_module_push_op(body: &[u8]) -> bool {
6900 serde_json::from_slice::<ControlOpProbe>(body)
6901 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6902 .unwrap_or(false)
6903}
6904
6905fn is_known_module_request_op(body: &[u8]) -> bool {
6906 serde_json::from_slice::<ControlOpProbe>(body)
6907 .map(|probe| is_module_to_subc_op(&probe.op))
6908 .unwrap_or(false)
6909}
6910
6911fn is_module_to_subc_op(op: &str) -> bool {
6912 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6913}
6914
6915fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6916 debug!(
6917 op = %op,
6918 connection_id = connection_id.get(),
6919 corr,
6920 "control dispatch"
6921 );
6922}
6923
6924fn log_slow_control_dispatch(
6925 dispatch_started_at: Option<StdInstant>,
6926 op: &'static str,
6927 connection_id: ConnectionId,
6928 corr: u64,
6929 route_open_timings: Option<&RouteOpenTimings>,
6930) {
6931 let Some(dispatch_started_at) = dispatch_started_at else {
6932 return;
6933 };
6934 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
6936 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6937 if let Some(timings) = route_open_timings {
6938 warn!(
6939 op = %op,
6940 connection_id = connection_id.get(),
6941 corr,
6942 elapsed_ms = elapsed.as_millis() as u64,
6943 module_id = ?timings.module_id,
6944 principal = %timings.principal,
6945 admission_ms = timings.admission.as_millis() as u64,
6946 project_root_ms = timings.project_root.as_millis() as u64,
6947 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
6948 bind_send_ms = timings.bind_send.as_millis() as u64,
6949 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
6950 "slow control dispatch"
6951 );
6952 return;
6953 }
6954 warn!(
6955 op = %op,
6956 connection_id = connection_id.get(),
6957 corr,
6958 elapsed_ms = elapsed.as_millis() as u64,
6959 "slow control dispatch"
6960 );
6961 }
6962}
6963
6964#[cfg(test)]
6966type TestCallerLookup =
6967 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
6968
6969fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
6971 match request {
6972 ClientControlRequest::SupervisorRestart { module_id, .. } => {
6973 Some(PrivilegedOperation::Restart { module_id })
6974 }
6975 ClientControlRequest::SupervisorSwap { module_id, .. } => {
6976 Some(PrivilegedOperation::Swap { module_id })
6977 }
6978 ClientControlRequest::SupervisorReload { module_id } => {
6979 Some(PrivilegedOperation::Reload { module_id })
6980 }
6981 ClientControlRequest::SupervisorRescan { preview } => {
6982 if *preview {
6983 None
6984 } else {
6985 Some(PrivilegedOperation::Rescan)
6986 }
6987 }
6988 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
6989 Some(PrivilegedOperation::SetEnabled {
6990 module_id,
6991 enabled: *enabled,
6992 })
6993 }
6994 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
6995 Some(PrivilegedOperation::ReleaseReserved { module_id })
6996 }
6997 ClientControlRequest::ServerDescribe {}
6998 | ClientControlRequest::CatalogList { .. }
6999 | ClientControlRequest::RouteOpen { .. }
7000 | ClientControlRequest::RoutePoll { .. }
7001 | ClientControlRequest::SupervisorList {}
7002 | ClientControlRequest::SupervisorHealth {}
7003 | ClientControlRequest::SupervisorHealthProbe { .. }
7004 | ClientControlRequest::SupervisorRoutes { .. }
7005 | ClientControlRequest::SupervisorProvenance { .. }
7006 | ClientControlRequest::SupervisorStderrTail { .. }
7007 | ClientControlRequest::SupervisorTerminals { .. }
7008 | ClientControlRequest::SupervisorSpawnSnapshot {}
7009 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7010 }
7011}
7012
7013fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7014 match request {
7015 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7016 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7017 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7018 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7019 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7020 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7021 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7022 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7023 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7024 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7025 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7026 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7027 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7028 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7029 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7030 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7031 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7032 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7033 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7034 }
7035}
7036
7037fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7043 use subc_protocol::scope::ScopeKind;
7044 const MAX_REF_CHARS: usize = 64;
7045 let kind = match stamp.kind {
7046 ScopeKind::Head => "head",
7047 ScopeKind::Worker => "worker",
7048 ScopeKind::Ephemeral => "ephemeral",
7049 };
7050 let mut scope_ref: String = stamp.scope_ref.chars().take(MAX_REF_CHARS).collect();
7051 if stamp.scope_ref.chars().nth(MAX_REF_CHARS).is_some() {
7052 scope_ref.push('…');
7053 }
7054 format!(
7055 "{kind}/{}/{scope_ref:?}@{}",
7056 principal_label(&stamp.owner),
7057 stamp.scope_epoch
7058 )
7059}
7060
7061fn principal_label(principal: &Principal) -> String {
7062 match principal {
7063 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7064 Principal::Direct => "direct".to_string(),
7065 other => format!("{other:?}"),
7066 }
7067}
7068
7069fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7070 match request {
7071 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7072 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7073 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7074 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7075 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7076 }
7077}
7078
7079fn parse_client_control_request(
7080 body: &[u8],
7081) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7082 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7083 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7084 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
7085 ControlRequestBodyError::InvalidBody
7086 }
7087 Ok(_) => ControlRequestBodyError::UnknownOp,
7088 Err(_) => ControlRequestBodyError::InvalidBody,
7089 };
7090 (err, classification)
7091 })
7092}
7093
7094fn parse_module_control_request_from_module(
7095 body: &[u8],
7096) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7097 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7098 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7099 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7100 Ok(_) => ControlRequestBodyError::UnknownOp,
7101 Err(_) => ControlRequestBodyError::InvalidBody,
7102 };
7103 (err, classification)
7104 })
7105}
7106
7107#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7108enum ProviderRoleKind {
7109 ToolProvider,
7110 PipelineStage,
7111 ManagementSurface,
7112 InternalService,
7113}
7114
7115fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7116 match role {
7117 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7118 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7119 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7120 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7121 }
7122}
7123
7124fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7125 roles.iter().map(provider_role_kind).collect()
7126}
7127
7128const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7131
7132#[derive(Debug, Default, PartialEq, Eq)]
7134struct ScopeOutcomeCounts {
7135 created: usize,
7136 replaced: usize,
7137 updated: usize,
7138 unchanged: usize,
7139 refused: usize,
7140}
7141
7142impl ScopeOutcomeCounts {
7143 fn of(results: &[ScopeRecordResult]) -> Self {
7144 let mut counts = Self::default();
7145 for result in results {
7146 let slot = match result.outcome {
7147 ScopeRecordOutcome::Created => &mut counts.created,
7148 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7149 ScopeRecordOutcome::Updated => &mut counts.updated,
7150 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7151 ScopeRecordOutcome::Refused => &mut counts.refused,
7152 };
7153 *slot += 1;
7154 }
7155 counts
7156 }
7157}
7158
7159#[cfg(test)]
7160mod scope_outcome_count_tests {
7161 use super::*;
7162
7163 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7164 ScopeRecordResult {
7165 scope_ref: "r".to_string(),
7166 scope_epoch: 1,
7167 outcome,
7168 code: None,
7169 message: None,
7170 version: None,
7171 parent_state: None,
7172 }
7173 }
7174
7175 #[test]
7178 fn every_outcome_is_counted_in_its_own_field() {
7179 let results = [
7180 result(ScopeRecordOutcome::Created),
7181 result(ScopeRecordOutcome::Created),
7182 result(ScopeRecordOutcome::Replaced),
7183 result(ScopeRecordOutcome::Updated),
7184 result(ScopeRecordOutcome::Unchanged),
7185 result(ScopeRecordOutcome::Refused),
7186 result(ScopeRecordOutcome::Refused),
7187 result(ScopeRecordOutcome::Refused),
7188 ];
7189 assert_eq!(
7190 ScopeOutcomeCounts::of(&results),
7191 ScopeOutcomeCounts {
7192 created: 2,
7193 replaced: 1,
7194 updated: 1,
7195 unchanged: 1,
7196 refused: 3,
7197 }
7198 );
7199 }
7200}
7201
7202fn capability_census_trigger(
7206 old: Option<&CapabilityDeclarations>,
7207 new: Option<&CapabilityDeclarations>,
7208) -> bool {
7209 let old_provides = old
7210 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7211 .unwrap_or_default();
7212 let old_denies = old
7213 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7214 .unwrap_or_default();
7215 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7216 provides: Vec::new(),
7217 requires: Vec::new(),
7218 must_never_reach: Vec::new(),
7219 });
7220
7221 new.provides
7222 .iter()
7223 .any(|capability| !old_provides.contains(capability))
7224 || new
7225 .must_never_reach
7226 .iter()
7227 .any(|capability| !old_denies.contains(capability))
7228}
7229
7230fn denied_capability<'a>(
7233 opening_manifest: &'a ModuleManifest,
7234 target_manifest: &ModuleManifest,
7235) -> Option<&'a str> {
7236 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7237 let target_capabilities = target_manifest.capabilities.as_ref()?;
7238 opening_capabilities
7239 .must_never_reach
7240 .iter()
7241 .find(|denied| {
7242 target_capabilities
7243 .provides
7244 .iter()
7245 .any(|provided| provided == *denied)
7246 })
7247 .map(String::as_str)
7248}
7249
7250fn catalog_update_frozen_field_message(
7251 registered: &ModuleManifest,
7252 provides: &[ProviderRole],
7253) -> Option<String> {
7254 let old_has_provides = !registered.provides.is_empty();
7255 let new_has_provides = !provides.is_empty();
7256 if old_has_provides != new_has_provides {
7257 return Some(format!(
7258 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7259 registered.module_id
7260 ));
7261 }
7262
7263 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7264 return Some(format!(
7265 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7266 registered.module_id
7267 ));
7268 }
7269
7270 let registered_concurrency = manifest_concurrency(registered);
7271 let mut candidate = registered.clone();
7272 candidate.provides = provides.to_vec();
7273 let candidate_concurrency = manifest_concurrency(&candidate);
7274 if candidate_concurrency != registered_concurrency {
7275 return Some(format!(
7276 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7277 registered.module_id, registered_concurrency, candidate_concurrency
7278 ));
7279 }
7280
7281 None
7284}
7285
7286fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7287 manifest.provides.iter().any(is_routable_role)
7288}
7289
7290fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7296 manifest
7297 .provides
7298 .iter()
7299 .find_map(|provider| match provider {
7300 ProviderRole::ToolProvider { concurrency, .. }
7301 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7302 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7303 })
7304 .unwrap_or(Concurrency::ModuleManaged)
7305}
7306
7307fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7314 let has_management_surface = manifest
7315 .provides
7316 .iter()
7317 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7318 if !has_management_surface {
7319 return false;
7320 }
7321 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7322 return false;
7323 };
7324 let Some(provides) = raw
7325 .get("manifest")
7326 .and_then(|manifest| manifest.get("provides"))
7327 .and_then(serde_json::Value::as_array)
7328 else {
7329 return false;
7330 };
7331 provides.iter().any(|role| {
7336 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7337 && role.get("concurrency").is_none()
7338 })
7339}
7340
7341fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7342 if peer_version != PROTOCOL_VERSION {
7343 return Err(format!(
7344 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7345 ));
7346 }
7347 Ok(PROTOCOL_VERSION)
7348}
7349
7350fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7351 Frame::build_with_version(
7352 response_version(frame),
7353 FrameType::Pong,
7354 frame.header.flags,
7355 0,
7356 0,
7357 frame.header.corr,
7358 Vec::new(),
7359 )
7360 .map_err(RouterError::FrameBuild)
7361}
7362
7363pub(crate) fn control_error_frame(
7364 frame: &Frame,
7365 code: &'static str,
7366 message: impl Into<String>,
7367) -> Result<Frame, RouterError> {
7368 control_error_body_frame(
7369 frame,
7370 ErrorBody {
7371 code: code.to_string(),
7372 message: message.into(),
7373 detail: None,
7374 },
7375 )
7376}
7377
7378fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7379 let body = serde_json::to_vec(&error).map_err(|err| {
7380 RouterError::backend(
7381 0,
7382 frame.header.corr,
7383 format!("failed to encode control ERROR: {err}"),
7384 )
7385 })?;
7386
7387 Frame::build_with_version(
7388 response_version(frame),
7389 FrameType::Error,
7390 control_flags(),
7391 0,
7392 0,
7393 frame.header.corr,
7394 body,
7395 )
7396 .map_err(RouterError::FrameBuild)
7397}
7398
7399fn control_response_body_frame<T: Serialize>(
7400 frame: &Frame,
7401 reply: &T,
7402 label: &'static str,
7403) -> Result<Frame, RouterError> {
7404 let body = serde_json::to_vec(reply).map_err(|err| {
7405 RouterError::backend(
7406 0,
7407 frame.header.corr,
7408 format!("failed to encode {label}: {err}"),
7409 )
7410 })?;
7411
7412 Frame::build_with_version(
7413 response_version(frame),
7414 FrameType::Response,
7415 control_flags(),
7416 0,
7417 0,
7418 frame.header.corr,
7419 body,
7420 )
7421 .map_err(RouterError::FrameBuild)
7422}
7423
7424fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7454 match err {
7455 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7456 ForwardingError::NoModuleConnection => "target_unavailable",
7457 ForwardingError::ModuleReloading { .. } => "module_reloading",
7458 ForwardingError::ClientRouteChannelExhausted { .. }
7459 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7460 ForwardingError::StaleModuleEndpoint
7461 | ForwardingError::UnknownReservation { .. }
7462 | ForwardingError::ConnectionClosing { .. }
7463 | ForwardingError::ClientEgressClosed { .. }
7464 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7465 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7468 ForwardingError::RelayCorrelationExhausted
7469 | ForwardingError::RouteOpenBuild(_)
7470 | ForwardingError::Poisoned => "forwarding_error",
7471 }
7472}
7473
7474fn response_version(frame: &Frame) -> u8 {
7475 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7476 frame.header.ver
7477 } else {
7478 PROTOCOL_VERSION
7479 }
7480}
7481
7482fn control_flags() -> Flags {
7483 Flags::new(false, Priority::Passive, false)
7484}
7485
7486fn send_goodbye_target_best_effort(
7491 counters: &DaemonCounters,
7492 target: &GoodbyeTarget,
7493 context: &'static str,
7494) {
7495 let Ok(frame) = Frame::build_with_version(
7496 target.negotiated_ver,
7497 FrameType::Goodbye,
7498 control_flags(),
7499 target.channel,
7500 target.epoch,
7501 0,
7502 Vec::new(),
7503 ) else {
7504 return;
7505 };
7506 crate::forwarding::send_module_route_goodbye(
7507 counters,
7508 &target.sink,
7509 frame,
7510 target.module_id.as_deref(),
7511 context,
7512 );
7513}
7514
7515pub(crate) fn send_route_control_pushes(
7516 forwarding: &ForwardingTable,
7517 routes: Vec<EndpointRoute>,
7518 push: ClientControlPush,
7519) {
7520 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7521 for route in routes {
7522 let target = route.goodbye_target;
7523 if let Some((existing, channels)) = targets
7524 .iter_mut()
7525 .find(|(existing, _)| existing.connection_id == target.connection_id)
7526 {
7527 debug_assert_eq!(
7528 existing.negotiated_ver, target.negotiated_ver,
7529 "one connection cannot negotiate multiple frame versions"
7530 );
7531 if !channels.contains(&target.channel) {
7532 channels.push(target.channel);
7533 }
7534 continue;
7535 }
7536 let channel = target.channel;
7537 targets.push((target, vec![channel]));
7538 }
7539 for (target, mut channels) in targets {
7540 channels.sort_unstable();
7541 let mut push = push.clone();
7542 match &mut push {
7543 ClientControlPush::RouteClosing {
7544 channels: covered, ..
7545 }
7546 | ClientControlPush::RouteClosed {
7547 channels: covered, ..
7548 } => *covered = channels,
7549 }
7550 let body = match serde_json::to_vec(&push) {
7551 Ok(body) => body,
7552 Err(err) => {
7553 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7554 continue;
7555 }
7556 };
7557 let frame = match Frame::build_with_version(
7558 target.negotiated_ver,
7559 FrameType::Push,
7560 control_flags(),
7561 0,
7562 0,
7563 0,
7564 body.clone(),
7565 ) {
7566 Ok(frame) => frame,
7567 Err(err) => {
7568 warn!(
7569 route_channel = target.channel,
7570 error = %err,
7571 "failed to build route lifecycle control PUSH frame"
7572 );
7573 continue;
7574 }
7575 };
7576 if let Err(err) = target.sink.try_send(frame) {
7577 if target.close_on_delivery_failure() {
7578 warn!(
7579 target_connection_id = target.connection_id.get(),
7580 route_channel = target.channel,
7581 error = %err,
7582 "route lifecycle control PUSH was not delivered to client; closing target connection"
7583 );
7584 let _ = forwarding.escalate_client_delivery_failure(
7585 target.connection_id,
7586 target.channel,
7587 target.epoch,
7588 CloseReason::new(
7589 "route_lifecycle_push_delivery_failed",
7590 format!(
7591 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7592 target.channel
7593 ),
7594 ),
7595 crate::forwarding::UndeliveredFrame {
7596 module_id: target.module_id.as_deref(),
7597 sink: &target.sink,
7598 },
7599 );
7600 }
7601 }
7602 }
7603}
7604
7605#[cfg(test)]
7606mod tests {
7607 mod activity_read {
7608 include!("activity_read_tests.rs");
7609 }
7610
7611 mod lifecycle_activity {
7612 include!("control_lifecycle_tests.rs");
7613 }
7614 #[cfg(unix)]
7615 #[tokio::test]
7616 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7617 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7618 let path = dir.join("subc.jsonc");
7619 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7620 "program":"/bin/sleep","args":["60"],"protocol":"none",
7621 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7622 }}}).to_string()).unwrap();
7623 let mut configured = crate::daemon_config::load(&path)
7624 .unwrap()
7625 .unwrap()
7626 .modules
7627 .pop()
7628 .unwrap();
7629 let registry = std::sync::Arc::new(crate::Registry::default());
7630 let handle = crate::SupervisorHandle::new();
7631 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7632 .with_handle(handle.clone());
7633 let module = supervisor
7634 .supervise_configured_with_health(
7635 configured.module_spec(),
7636 true,
7637 configured.health.clone(),
7638 None,
7639 configured.restart,
7640 )
7641 .unwrap();
7642 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7643 let before = module.status().unwrap().pid;
7644 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7645 configured.health.cadence = std::time::Duration::from_secs(3600);
7646 let health_only = handler
7647 .reconcile_supervised_modules(
7648 &supervisor,
7649 vec![configured.clone()],
7650 false,
7651 Principal::Direct,
7652 )
7653 .await
7654 .unwrap();
7655 assert!(
7656 health_only.changed_pending_reload.is_empty(),
7657 "health policy is already applied live"
7658 );
7659 assert_eq!(module.status().unwrap().pid, before);
7660 assert_eq!(
7661 module.configuration().unwrap().1.http,
7662 configured.health.http
7663 );
7664 configured.args = vec!["61".into()];
7665 let launch = handler
7666 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7667 .await
7668 .unwrap();
7669 assert_eq!(launch.changed_pending_reload, ["stock"]);
7670 assert_eq!(
7671 module.status().unwrap().pid,
7672 before,
7673 "a launch edit is stored until reload"
7674 );
7675 module.drain().await.unwrap();
7676 }
7677 use cortexkit_test_support::ScratchDir;
7678 use std::{
7679 collections::BTreeMap,
7680 fmt,
7681 path::PathBuf,
7682 sync::{Arc, Mutex},
7683 time::Duration,
7684 };
7685
7686 use serde_json::{json, Value};
7687 use subc_protocol::{
7688 manifest::{
7689 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7690 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7691 },
7692 session::HealthStatus,
7693 FrameType,
7694 };
7695
7696 use super::*;
7697 use crate::{
7698 forwarding::{DataRoute, DataRouteState},
7699 registry::ChannelState,
7700 router::FrameSink,
7701 stderr_tail::DEFAULT_MAX_LINE_BYTES,
7702 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7703 RouteCtx, Router,
7704 };
7705 use tokio::{
7706 sync::mpsc,
7707 time::{sleep, Instant},
7708 };
7709 use tracing::{
7710 field::{Field, Visit},
7711 Event, Subscriber,
7712 };
7713 use tracing_subscriber::{layer::Context, prelude::*, Layer};
7714
7715 fn fake_aft_stub_path() -> PathBuf {
7734 let mut path = std::env::current_exe().expect("current_exe available in tests");
7735 path.pop(); path.pop(); path.push(if cfg!(windows) {
7738 "fake-aft-stub.exe"
7739 } else {
7740 "fake-aft-stub"
7741 });
7742 assert!(
7743 path.exists(),
7744 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7745 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7746 path.display()
7747 );
7748 path
7749 }
7750
7751 fn client_retries(code: &str) -> bool {
7756 subc_protocol::error_codes::is_retryable_route_open(code)
7757 }
7758
7759 #[test]
7770 fn retryability_of_forwarding_codes_matches_the_failure() {
7771 let transient = [
7774 ForwardingError::NoModuleConnection,
7775 ForwardingError::ModuleReloading {
7776 module_id: "m".into(),
7777 },
7778 ForwardingError::StaleModuleEndpoint,
7779 ForwardingError::UnknownReservation {
7780 client_channel: 1,
7781 module_channel: 1,
7782 },
7783 ForwardingError::ConnectionClosing {
7784 connection_id: ConnectionId::new(1),
7785 },
7786 ForwardingError::ClientEgressClosed {
7787 connection_id: ConnectionId::new(1),
7788 },
7789 ForwardingError::ModuleEgressUnavailable {
7790 connection_id: ConnectionId::new(1),
7791 },
7792 ];
7793 for err in transient {
7794 let code = forwarding_error_code(&err);
7795 assert!(
7796 client_retries(code),
7797 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7798 );
7799 }
7800
7801 let permanent = [
7806 ForwardingError::ConnectionRoleConflict {
7807 connection_id: ConnectionId::new(1),
7808 },
7809 ForwardingError::ClientRouteChannelExhausted {
7810 connection_id: ConnectionId::new(1),
7811 },
7812 ForwardingError::ModuleRouteChannelExhausted {
7813 endpoint: ModuleEndpointId {
7814 connection_id: ConnectionId::new(1),
7815 generation: 1,
7816 },
7817 },
7818 ForwardingError::RelayCorrelationExhausted,
7819 ForwardingError::RouteOpenBuild("x".into()),
7820 ForwardingError::Poisoned,
7821 ];
7822 for err in permanent {
7823 let code = forwarding_error_code(&err);
7824 assert!(
7825 !client_retries(code),
7826 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7827 );
7828 }
7829 }
7830
7831 #[tokio::test]
7859 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7860 let handler = ControlHandler::default();
7861 let frame =
7862 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7863
7864 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7868 assert_eq!(
7869 stamped,
7870 Principal::Direct,
7871 "a caller that proved nothing must not be stamped as a supervised module"
7872 );
7873
7874 let forged = handler
7879 .route_open_principal(
7880 &frame,
7881 Some(ConsumerIdentity {
7882 module_id: "aft".to_string(),
7883 launch_nonce: "not-a-real-nonce".to_string(),
7884 }),
7885 )
7886 .unwrap();
7887 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7888 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7889 }
7890
7891 #[test]
7908 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7909 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7910 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7911 let ClientControlRequest::RouteOpen {
7912 consumer_identity, ..
7913 } = parsed
7914 else {
7915 panic!("route.open body must parse as RouteOpen");
7916 };
7917 assert_eq!(
7918 consumer_identity,
7919 Some(ConsumerIdentity {
7920 module_id: "aft".to_string(),
7921 launch_nonce: "n".to_string(),
7922 }),
7923 "the wire field name must reach the value route_open_principal reads"
7924 );
7925 }
7926
7927 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7928 ModuleManifest::builder(module_id, "0.1.0")
7929 .protocol_ver(protocol_ver)
7930 .provides(vec![ProviderRole::ToolProvider {
7931 tools: vec![Tool {
7932 name: "read".to_string(),
7933 description: None,
7934 execution_mode: ExecutionMode::Pure,
7935 schema: json!({"type": "object"}),
7936 }],
7937 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7938 concurrency: Concurrency::ModuleManaged,
7939 emits_push: true,
7940 sub_supervises: true,
7941 }])
7942 .build()
7943 }
7944
7945 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7946 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7947 }
7948
7949 fn hello_frame_with_control_ops(
7950 module_id: &str,
7951 protocol_ver: u8,
7952 corr: u64,
7953 control_ops: Option<Vec<String>>,
7954 ) -> Frame {
7955 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7956 }
7957
7958 fn hello_frame_with_nonce(
7959 module_id: &str,
7960 protocol_ver: u8,
7961 corr: u64,
7962 launch_nonce: Option<&str>,
7963 ) -> Frame {
7964 hello_frame_full(
7965 module_id,
7966 protocol_ver,
7967 corr,
7968 None,
7969 launch_nonce.map(ToOwned::to_owned),
7970 )
7971 }
7972
7973 fn hello_frame_full(
7974 module_id: &str,
7975 protocol_ver: u8,
7976 corr: u64,
7977 control_ops: Option<Vec<String>>,
7978 launch_nonce: Option<String>,
7979 ) -> Frame {
7980 let body = serde_json::to_vec(&ModuleHelloBody {
7981 manifest: manifest(module_id, protocol_ver),
7982 protocol_ver,
7983 control_ops,
7984 launch_nonce,
7985 })
7986 .unwrap();
7987 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7988 }
7989
7990 fn non_routable_hello_frame_with_control_ops(
7991 module_id: &str,
7992 corr: u64,
7993 control_ops: Option<Vec<String>>,
7994 ) -> Frame {
7995 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
7996 manifest.provides.clear();
7997 let body = serde_json::to_vec(&ModuleHelloBody {
7998 manifest,
7999 protocol_ver: PROTOCOL_VERSION,
8000 control_ops,
8001 launch_nonce: None,
8002 })
8003 .unwrap();
8004 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8005 }
8006
8007 fn capability_grammar_hello_frame(
8008 capabilities: Value,
8009 runtime_computed: Option<Value>,
8010 corr: u64,
8011 ) -> Frame {
8012 let mut body = serde_json::to_value(ModuleHelloBody {
8013 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8014 protocol_ver: PROTOCOL_VERSION,
8015 control_ops: None,
8016 launch_nonce: None,
8017 })
8018 .expect("HELLO body serializes");
8019 body["manifest"]["capabilities"] = capabilities;
8020 if let Some(runtime_computed) = runtime_computed {
8021 body["runtime_computed"] = runtime_computed;
8022 }
8023 Frame::build(
8024 FrameType::Hello,
8025 control_flags(),
8026 0,
8027 0,
8028 corr,
8029 serde_json::to_vec(&body).expect("HELLO body reserializes"),
8030 )
8031 .expect("HELLO frame builds")
8032 }
8033
8034 fn channel_request(channel: u16, corr: u64) -> Frame {
8035 Frame::build(
8036 FrameType::Request,
8037 Flags::new(true, Priority::Interactive, false),
8038 channel,
8039 0,
8040 corr,
8041 b"opaque".to_vec(),
8042 )
8043 .unwrap()
8044 }
8045
8046 fn route_ctx(
8047 connection_id: ConnectionId,
8048 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8049 let (tx, rx) = mpsc::channel(8);
8050 (
8051 RouteCtx {
8052 connection_id,
8053 egress: FrameSink::new(tx),
8054 },
8055 rx,
8056 )
8057 }
8058
8059 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8060 serde_json::from_slice(&frame.body).unwrap()
8061 }
8062
8063 async fn hello_via_sink(
8068 handler: &ControlHandler,
8069 ctx: &RouteCtx,
8070 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8071 hello: Frame,
8072 ) -> Frame {
8073 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8074 assert!(
8075 replies.is_empty(),
8076 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8077 );
8078 let ack = rx
8079 .try_recv()
8080 .expect("HELLO_ACK is queued on the module sink")
8081 .frame;
8082 assert_eq!(ack.header.ty, FrameType::HelloAck);
8083 ack
8084 }
8085
8086 fn parse_error(frame: &Frame) -> Value {
8087 serde_json::from_slice(&frame.body).unwrap()
8088 }
8089
8090 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8091 serde_json::from_slice(&frame.body).unwrap()
8092 }
8093
8094 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8095 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8096 route_channel,
8097 route_epoch: 0,
8098 kind,
8099 })
8100 .unwrap();
8101 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8102 }
8103
8104 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8105 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8106 module_id: module_id.to_string(),
8107 })
8108 .unwrap();
8109 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8110 }
8111
8112 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8113 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8114 }
8115
8116 fn route_open_frame_with_consumer_capabilities(
8117 corr: u64,
8118 module_id: &str,
8119 project_root: ScratchDir,
8120 consumer_capabilities: Option<Vec<String>>,
8121 ) -> Frame {
8122 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8123 target: RouteTarget::ToolProvider {
8124 module_id: module_id.to_string(),
8125 },
8126 identity: BindIdentity::new(
8127 project_root.path().to_path_buf(),
8128 "unit".to_string(),
8129 "session".to_string(),
8130 ),
8131 consumer_identity: None,
8132 consumer_capabilities,
8133 role_versions: None,
8134 admission_facts: None,
8135 scope: None,
8136 })
8137 .unwrap();
8138 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8139 }
8140
8141 fn route_open_frame_with_role_versions(
8142 corr: u64,
8143 module_id: &str,
8144 project_root: ScratchDir,
8145 role_versions: Option<BTreeMap<String, String>>,
8146 ) -> Frame {
8147 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8148 target: RouteTarget::ToolProvider {
8149 module_id: module_id.to_string(),
8150 },
8151 identity: BindIdentity::new(
8152 project_root.path().to_path_buf(),
8153 "unit".to_string(),
8154 format!("session-{corr}"),
8155 ),
8156 consumer_identity: None,
8157 consumer_capabilities: None,
8158 role_versions,
8159 admission_facts: None,
8160 scope: None,
8161 })
8162 .unwrap();
8163 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8164 }
8165
8166 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8167 entries
8168 .iter()
8169 .map(|(role, version)| (role.to_string(), version.to_string()))
8170 .collect()
8171 }
8172
8173 fn route_open_frame_with_admission_facts(
8174 corr: u64,
8175 module_id: &str,
8176 project_root: ScratchDir,
8177 consumer_identity: Option<subc_control::ConsumerIdentity>,
8178 facts: Option<Value>,
8179 ) -> Frame {
8180 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8181 target: RouteTarget::ToolProvider {
8182 module_id: module_id.to_string(),
8183 },
8184 identity: BindIdentity::new(
8185 project_root.path().to_path_buf(),
8186 "unit".to_string(),
8187 format!("session-{corr}"),
8188 ),
8189 consumer_identity,
8190 consumer_capabilities: None,
8191 role_versions: None,
8192 admission_facts: facts,
8193 scope: None,
8194 })
8195 .unwrap();
8196 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8197 }
8198
8199 #[derive(Clone, Default)]
8200 struct EventCapture {
8201 events: Arc<Mutex<Vec<CapturedEvent>>>,
8202 }
8203
8204 #[derive(Clone, Debug)]
8205 struct CapturedEvent {
8206 target: String,
8207 level: tracing::Level,
8208 fields: BTreeMap<String, String>,
8209 }
8210
8211 impl EventCapture {
8212 fn events(&self) -> Vec<CapturedEvent> {
8213 self.events.lock().unwrap().clone()
8214 }
8215 }
8216
8217 impl<S> Layer<S> for EventCapture
8218 where
8219 S: Subscriber,
8220 {
8221 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8222 let mut visitor = EventFieldVisitor::default();
8223 event.record(&mut visitor);
8224 self.events.lock().unwrap().push(CapturedEvent {
8225 target: event.metadata().target().to_string(),
8226 level: *event.metadata().level(),
8227 fields: visitor.fields,
8228 });
8229 }
8230 }
8231
8232 #[derive(Default)]
8233 struct EventFieldVisitor {
8234 fields: BTreeMap<String, String>,
8235 }
8236
8237 impl Visit for EventFieldVisitor {
8238 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8239 self.fields
8240 .insert(field.name().to_string(), format!("{value:?}"));
8241 }
8242 }
8243
8244 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8245 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8246 status,
8247 detail: Some("warming".to_string()),
8248 metrics: Some(json!({"queue_depth": 3})),
8249 })
8250 .unwrap();
8251 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8252 }
8253
8254 fn route_bind_ack(corr: u64) -> Frame {
8255 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8256 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8257 }
8258
8259 fn unique_project_root(label: &str) -> ScratchDir {
8260 ScratchDir::new(label)
8261 }
8262
8263 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8264 match parse_route_poll(frame) {
8265 ClientControlResponse::RoutePoll {
8266 status: None,
8267 live: Some(live),
8268 ..
8269 } => assert_eq!(live, expected_live),
8270 other => panic!("unexpected route.poll response: {other:?}"),
8271 }
8272 }
8273
8274 fn bind_liveness_route(
8275 registry: &Registry,
8276 forwarding: &ForwardingTable,
8277 module_id: &str,
8278 ) -> (RouteCtx, u16, u32) {
8279 let module_connection = ConnectionId::new(101);
8280 let client_connection = ConnectionId::new(202);
8281 let registration = registry
8282 .register_with_control_ops(
8283 manifest(module_id, PROTOCOL_VERSION),
8284 PROTOCOL_VERSION,
8285 module_connection,
8286 module_baseline_control_ops(),
8287 )
8288 .unwrap();
8289 let (module_tx, _module_rx) = mpsc::channel(8);
8290 let endpoint = forwarding
8291 .register_module_connection(
8292 module_connection,
8293 module_id.to_string(),
8294 PROTOCOL_VERSION,
8295 manifest_concurrency(®istration.manifest),
8296 FrameSink::new(module_tx),
8297 )
8298 .unwrap();
8299 let (client_ctx, _client_rx) = route_ctx(client_connection);
8300 let pending = forwarding
8301 .begin_route_bind_relay_for_test(
8302 client_connection,
8303 client_ctx.egress.clone(),
8304 1,
8305 module_id,
8306 )
8307 .unwrap();
8308 assert_eq!(pending.endpoint, endpoint);
8309 let route_channel = pending.client_channel;
8310 let route_epoch = pending.client_epoch;
8311 forwarding
8312 .complete_pending_relay(
8313 module_connection,
8314 pending.corr,
8315 RouteBindRelayOutcome::Accepted,
8316 )
8317 .unwrap();
8318 (client_ctx, route_channel, route_epoch)
8319 }
8320
8321 struct FakeProcessLiveness {
8322 live: Option<bool>,
8323 }
8324
8325 impl ModuleProcessLiveness for FakeProcessLiveness {
8326 fn process_live(&self, _module_id: &str) -> Option<bool> {
8327 self.live
8328 }
8329 }
8330
8331 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8332 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8333 {
8334 let registry = Arc::new(Registry::default());
8335 let supervisor_handle = SupervisorHandle::new();
8336 let supervisor = Supervisor::new_for_test(
8337 Arc::clone(®istry),
8338 RestartPolicy::new(1, Duration::from_millis(10)),
8339 )
8340 .with_handle(supervisor_handle.clone());
8341 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8342 let module = supervisor
8343 .spawn(ModuleSpec {
8344 module_id: "stderr-tail-wire".to_string(),
8345 program: fake_aft_stub_path(),
8346 args: Vec::new(),
8347 env: vec![
8348 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8349 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8350 ],
8351 reserved: false,
8352 reserved_prefixes: Vec::new(),
8353 protocol: ModuleProtocol::Subc,
8354 overlap: Default::default(),
8355 })
8356 .unwrap();
8357
8358 let deadline = Instant::now() + Duration::from_secs(5);
8359 loop {
8360 let tail = module.stderr_tail(None, None);
8361 if tail
8362 .entries
8363 .iter()
8364 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8365 && tail.entries.iter().any(|entry| {
8366 matches!(
8367 entry,
8368 TailEntry::Line {
8369 truncated: true,
8370 ..
8371 }
8372 )
8373 })
8374 {
8375 break;
8376 }
8377 assert!(
8378 Instant::now() < deadline,
8379 "module did not produce a truncated line and restart boundary: {tail:?}"
8380 );
8381 sleep(Duration::from_millis(10)).await;
8382 }
8383
8384 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8385 let request = ClientControlRequest::SupervisorStderrTail {
8386 module_id: "stderr-tail-wire".to_string(),
8387 max_lines: None,
8388 max_bytes: None,
8389 };
8390 let frame = Frame::build(
8391 FrameType::Request,
8392 control_flags(),
8393 0,
8394 0,
8395 1,
8396 serde_json::to_vec(&request).unwrap(),
8397 )
8398 .unwrap();
8399 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8400 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8401 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8402 serde_json::from_slice(&responses[0].body).unwrap()
8403 else {
8404 panic!("expected supervisor.stderr_tail response");
8405 };
8406
8407 assert!(
8408 tail.entries
8409 .iter()
8410 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8411 "the control response lost the restart boundary"
8412 );
8413 let Some(StderrTailEntry::Line {
8414 text,
8415 truncated,
8416 at_ms,
8417 }) = tail.entries.iter().find(|entry| {
8418 matches!(
8419 entry,
8420 StderrTailEntry::Line {
8421 truncated: true,
8422 ..
8423 }
8424 )
8425 })
8426 else {
8427 panic!("the control response lost the truncated line");
8428 };
8429 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8430 assert!(*truncated);
8431 assert!(
8432 at_ms.is_some(),
8433 "the control response lost the line's capture time"
8434 );
8435 }
8436
8437 #[tokio::test(flavor = "current_thread")]
8442 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8443 let dir = ScratchDir::new("terminals-off-worker");
8444 let journal_path = dir.join("terminals.jsonl");
8445 let registry = Arc::new(Registry::default());
8446 let supervisor_handle = SupervisorHandle::new();
8447 let supervisor =
8448 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8449 .with_handle(supervisor_handle.clone())
8450 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8451 let module = supervisor
8452 .spawn(ModuleSpec {
8453 module_id: "terminal-off-worker".to_string(),
8454 program: fake_aft_stub_path(),
8455 args: Vec::new(),
8456 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8457 reserved: false,
8458 reserved_prefixes: Vec::new(),
8459 protocol: ModuleProtocol::Subc,
8460 overlap: Default::default(),
8461 })
8462 .unwrap();
8463 let deadline = Instant::now() + Duration::from_secs(5);
8464 while module.terminal_history().entries.len() != 2 {
8465 assert!(Instant::now() < deadline, "module did not record two exits");
8466 sleep(Duration::from_millis(10)).await;
8467 }
8468
8469 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8470 let handler =
8471 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8472 let frame = Frame::build(
8473 FrameType::Request,
8474 control_flags(),
8475 0,
8476 0,
8477 1,
8478 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8479 module_id: "terminal-off-worker".to_string(),
8480 })
8481 .unwrap(),
8482 )
8483 .unwrap();
8484 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8485 let spawned_at = std::time::Instant::now();
8486 let read = tokio::spawn({
8487 let handler = Arc::clone(&handler);
8488 async move { handler.handle_control_frame(&ctx, frame).await }
8489 });
8490 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8493 .await
8494 .unwrap()
8495 .expect("the history read reached its pause");
8496 let elapsed = spawned_at.elapsed();
8497 assert!(
8498 elapsed < Duration::from_secs(2) && !read.is_finished(),
8499 "this task could not run while the history read was paused \
8500 (resumed after {elapsed:?}, read finished: {})",
8501 read.is_finished()
8502 );
8503
8504 drop(release);
8505 let responses = read.await.unwrap().unwrap();
8506 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8507 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8508 panic!("expected supervisor.terminals response");
8509 };
8510 assert_eq!(terminals.entries.len(), 2);
8511 assert_eq!(terminals.journal_skipped_lines, 0);
8512 assert_eq!(terminals.journal_read_errors, 0);
8513 }
8514
8515 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8516 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8517 let registry = Arc::new(Registry::default());
8518 let supervisor_handle = SupervisorHandle::new();
8519 let supervisor =
8520 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8521 .with_handle(supervisor_handle.clone());
8522 let module = supervisor
8523 .spawn(ModuleSpec {
8524 module_id: "terminal-golden".to_string(),
8525 program: fake_aft_stub_path(),
8526 args: Vec::new(),
8527 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8528 reserved: false,
8529 reserved_prefixes: Vec::new(),
8530 protocol: ModuleProtocol::Subc,
8531 overlap: Default::default(),
8532 })
8533 .unwrap();
8534
8535 let deadline = Instant::now() + Duration::from_secs(5);
8536 while module.terminal_history().entries.len() != 2 {
8537 assert!(
8538 Instant::now() < deadline,
8539 "module did not retain two terminal exits: {:?}",
8540 module.terminal_history()
8541 );
8542 sleep(Duration::from_millis(10)).await;
8543 }
8544
8545 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8546 let request = ClientControlRequest::SupervisorTerminals {
8547 module_id: "terminal-golden".to_string(),
8548 };
8549 let frame = Frame::build(
8550 FrameType::Request,
8551 control_flags(),
8552 0,
8553 0,
8554 1,
8555 serde_json::to_vec(&request).unwrap(),
8556 )
8557 .unwrap();
8558 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8559 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8560 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8561 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8562 panic!("expected supervisor.terminals response");
8563 };
8564 assert_eq!(terminals.entries.len(), 2);
8565 assert_eq!(terminals.dropped, 0);
8566
8567 let mut rendered = serde_json::to_value(response).unwrap();
8568 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8571 for (index, entry) in rendered["entries"]
8572 .as_array_mut()
8573 .expect("terminal response entries array")
8574 .iter_mut()
8575 .enumerate()
8576 {
8577 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8578 }
8579
8580 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8581 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8582 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8583 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8584 std::fs::write(&golden_path, &serialized).unwrap();
8585 }
8586 let expected: Value =
8587 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8588 assert_eq!(rendered, expected);
8589 }
8590
8591 #[test]
8592 fn hello_registers_manifest_and_returns_ack() {
8593 let registry = Arc::new(Registry::default());
8594 let handler = ControlHandler::new(Arc::clone(®istry));
8595 let conn = ConnectionId::new(1);
8596
8597 let responses = handler
8598 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8599 .unwrap();
8600
8601 assert_eq!(responses.len(), 1);
8602 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8603 assert_eq!(responses[0].header.channel, 0);
8604 assert_eq!(responses[0].header.corr, 7);
8605 let ack = parse_ack(&responses[0]);
8606 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8607 assert!(ack
8608 .subc_capabilities
8609 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8610 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8611 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8612 assert!(ack
8613 .subc_ops
8614 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8615 assert!(ack
8616 .subc_ops
8617 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8618
8619 let registration = registry.get_module("aft").unwrap().unwrap();
8620 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8621 assert_eq!(registration.state, ChannelState::Active);
8622 assert_eq!(registration.connection_id, conn);
8623 assert_eq!(registration.control_ops, module_baseline_control_ops());
8624 }
8625
8626 #[test]
8627 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8628 let invalid_identifiers = [
8629 ("case_change", "credentials-Provider/v1"),
8630 ("leading_zero", "credentials-provider/v01"),
8631 ("trailing_hyphen", "credentials-provider-/v1"),
8632 ("consecutive_hyphens", "credentials--provider/v1"),
8633 ("uppercase", "Credentials-provider/v1"),
8634 ("missing_v", "credentials-provider/1"),
8635 ("whitespace", "credentials provider/v1"),
8636 ("zero_version", "credentials-provider/v0"),
8637 ("out_of_range_version", "credentials-provider/v4294967296"),
8638 (
8639 "overlength_name",
8640 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8641 ),
8642 ];
8643 let mut cases = invalid_identifiers
8644 .into_iter()
8645 .map(|(name, identifier)| {
8646 (
8647 format!("identifier_{name}"),
8648 "capabilities.provides[0]".to_string(),
8649 identifier.to_string(),
8650 json!({ "provides": [identifier] }),
8651 None,
8652 )
8653 })
8654 .collect::<Vec<_>>();
8655 cases.extend([
8656 (
8657 "unknown_need".to_string(),
8658 "capabilities.requires[0].need".to_string(),
8659 "deferred".to_string(),
8660 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8661 None,
8662 ),
8663 (
8664 "duplicate_provides".to_string(),
8665 "capabilities.provides[1]".to_string(),
8666 "credentials-provider/v1".to_string(),
8667 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8668 None,
8669 ),
8670 (
8671 "duplicate_must_never_reach".to_string(),
8672 "capabilities.must_never_reach[1]".to_string(),
8673 "credentials-provider/v1".to_string(),
8674 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8675 None,
8676 ),
8677 (
8678 "duplicate_requires_same_need".to_string(),
8679 "capabilities.requires[1]".to_string(),
8680 "credentials-provider/v1".to_string(),
8681 json!({ "requires": [
8682 { "capability": "credentials-provider/v1", "need": "required" },
8683 { "capability": "credentials-provider/v1", "need": "required" }
8684 ] }),
8685 None,
8686 ),
8687 (
8688 "duplicate_requires_conflicting_need".to_string(),
8689 "capabilities.requires[1]".to_string(),
8690 "credentials-provider/v1".to_string(),
8691 json!({ "requires": [
8692 { "capability": "credentials-provider/v1", "need": "required" },
8693 { "capability": "credentials-provider/v1", "need": "optional" }
8694 ] }),
8695 None,
8696 ),
8697 (
8698 "capabilities_root_pointer".to_string(),
8699 "runtime_computed[0]".to_string(),
8700 "/capabilities".to_string(),
8701 json!({}),
8702 Some(json!(["/capabilities"])),
8703 ),
8704 (
8705 "capabilities_descendant_pointer".to_string(),
8706 "runtime_computed[0]".to_string(),
8707 "/capabilities/provides".to_string(),
8708 json!({}),
8709 Some(json!(["/capabilities/provides"])),
8710 ),
8711 (
8712 "malformed_pointer_without_leading_slash".to_string(),
8713 "runtime_computed[0]".to_string(),
8714 "capabilities".to_string(),
8715 json!({}),
8716 Some(json!(["capabilities"])),
8717 ),
8718 (
8719 "malformed_pointer_escape".to_string(),
8720 "runtime_computed[0]".to_string(),
8721 "/roles/~2/tools".to_string(),
8722 json!({}),
8723 Some(json!(["/roles/~2/tools"])),
8724 ),
8725 (
8726 "unknown_capabilities_field".to_string(),
8727 "capabilities.future".to_string(),
8728 "<array>".to_string(),
8729 json!({ "future": [] }),
8730 None,
8731 ),
8732 ]);
8733
8734 for (index, (name, field, value, capabilities, runtime_computed)) in
8735 cases.into_iter().enumerate()
8736 {
8737 let registry = Arc::new(Registry::default());
8738 let handler = ControlHandler::new(Arc::clone(®istry));
8739 let response = handler
8740 .handle_control(
8741 ConnectionId::new((index + 1) as u64),
8742 capability_grammar_hello_frame(
8743 capabilities,
8744 runtime_computed,
8745 index as u64 + 1,
8746 ),
8747 )
8748 .expect("invalid HELLO returns a refusal");
8749
8750 assert_eq!(response.len(), 1, "{name} must emit one refusal");
8751 let error = parse_error(&response[0]);
8752 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8753 let message = error["message"]
8754 .as_str()
8755 .expect("error message is a string");
8756 assert!(
8757 message.contains(&field),
8758 "{name}: field missing from {message}"
8759 );
8760 assert!(
8761 message.contains(&value),
8762 "{name}: value missing from {message}"
8763 );
8764 assert_eq!(
8765 registry
8766 .active_registration_count()
8767 .expect("registry reads"),
8768 0,
8769 "{name}: refused HELLO must not create a catalog entry"
8770 );
8771 }
8772 }
8773
8774 #[test]
8775 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8776 let registry = Arc::new(Registry::default());
8777 let handler = ControlHandler::new(Arc::clone(®istry));
8778 let capabilities = json!({
8779 "provides": ["credentials-provider/v1"],
8780 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8781 "must_never_reach": ["federation-transport/v1"]
8782 });
8783 let response = handler
8784 .handle_control(
8785 ConnectionId::new(99),
8786 capability_grammar_hello_frame(
8787 capabilities.clone(),
8788 Some(json!(["/roles/0/tools"])),
8789 99,
8790 ),
8791 )
8792 .expect("valid HELLO registers");
8793 assert_eq!(response[0].header.ty, FrameType::HelloAck);
8794
8795 let request = Frame::build(
8796 FrameType::Request,
8797 control_flags(),
8798 0,
8799 0,
8800 100,
8801 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8802 .expect("catalog request serializes"),
8803 )
8804 .expect("catalog request frame builds");
8805 let response = handler
8806 .handle_catalog_list(request, None)
8807 .expect("catalog list succeeds");
8808 let ClientControlResponse::CatalogList { modules, .. } =
8809 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8810 else {
8811 panic!("catalog request must return catalog.list");
8812 };
8813 assert_eq!(modules.len(), 1);
8814 assert_eq!(
8815 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8816 capabilities
8817 );
8818 }
8819
8820 #[test]
8821 fn catalog_list_mirrors_management_operation_description() {
8822 let registry = Arc::new(Registry::default());
8823 let handler = ControlHandler::new(Arc::clone(®istry));
8824 let description = "List managed records and return their identifiers and metadata.";
8825 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8826 manifest.provides = vec![ProviderRole::ManagementSurface {
8827 operations: vec![ManagementOperation {
8828 name: "records.list".to_string(),
8829 kind: ManagementOperationKind::Query,
8830 description: Some(description.to_string()),
8831 }],
8832 config_schema: json!({"type": "object"}),
8833 observability: vec![ObservabilitySurface {
8834 name: "records.stats".to_string(),
8835 kind: ObservabilityKind::Snapshot,
8836 }],
8837 identity_scope: vec![IdentityScope::Project],
8838 concurrency: Concurrency::ModuleManaged,
8839 }];
8840 registry
8841 .register_with_control_ops(
8842 manifest,
8843 PROTOCOL_VERSION,
8844 ConnectionId::new(99),
8845 Vec::new(),
8846 )
8847 .expect("described management manifest registers");
8848
8849 let request = Frame::build(
8850 FrameType::Request,
8851 control_flags(),
8852 0,
8853 0,
8854 100,
8855 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8856 .expect("catalog request serializes"),
8857 )
8858 .expect("catalog request frame builds");
8859 let response = handler
8860 .handle_catalog_list(request, None)
8861 .expect("catalog list succeeds");
8862 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8863 assert_eq!(
8864 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8865 "catalog.list must preserve the declared operation description verbatim"
8866 );
8867 }
8868
8869 #[test]
8870 fn catalog_list_mirrors_registered_operation_delivery() {
8871 let registry = Arc::new(Registry::default());
8872 let handler = ControlHandler::new(Arc::clone(®istry));
8873 let delivery =
8874 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
8875 let mut body = serde_json::to_value(ModuleHelloBody {
8876 manifest: manifest("watch-management", PROTOCOL_VERSION),
8877 protocol_ver: PROTOCOL_VERSION,
8878 control_ops: None,
8879 launch_nonce: None,
8880 })
8881 .unwrap();
8882 body["manifest"]["provides"] = json!([{
8883 "role": "management_surface",
8884 "operations": [
8885 {"name": "status", "kind": "query"},
8886 {"name": "future", "kind": "query"},
8887 {"name": "reset", "kind": "mutate"}
8888 ],
8889 "config_schema": {}, "observability": [], "identity_scope": []
8890 }]);
8891 body["manifest"]["operation_delivery"] = delivery.clone();
8892 let hello = Frame::build(
8893 FrameType::Hello,
8894 control_flags(),
8895 0,
8896 0,
8897 99,
8898 serde_json::to_vec(&body).unwrap(),
8899 )
8900 .unwrap();
8901 let response = handler
8902 .handle_control(ConnectionId::new(99), hello)
8903 .unwrap();
8904 assert_eq!(response[0].header.ty, FrameType::HelloAck);
8905
8906 let (_, registrations) = registry.list_modules().unwrap();
8907 let registered = ®istrations[0].manifest;
8908 assert!(registered.is_watch_operation("status"));
8909 assert!(!registered.is_watch_operation("future"));
8910 assert!(!registered.is_watch_operation("reset"));
8911 assert!(!registered.is_watch_operation("missing"));
8912
8913 let request = Frame::build(
8914 FrameType::Request,
8915 control_flags(),
8916 0,
8917 0,
8918 100,
8919 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
8920 )
8921 .unwrap();
8922 let response = handler.handle_catalog_list(request, None).unwrap();
8923 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
8924 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
8925 let ClientControlResponse::CatalogList { modules, .. } =
8926 serde_json::from_slice(&response[0].body).unwrap()
8927 else {
8928 panic!("expected catalog.list");
8929 };
8930 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
8931 }
8932
8933 #[test]
8934 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8935 let registry = Arc::new(Registry::default());
8936 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8937 [("vault".to_string(), true), ("squatter".to_string(), true)],
8938 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8939 );
8940 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8941 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8942 provides: vec!["credentials-provider/v1".to_string()],
8943 requires: Vec::new(),
8944 must_never_reach: Vec::new(),
8945 });
8946 let frame = Frame::build(
8947 FrameType::Hello,
8948 control_flags(),
8949 0,
8950 0,
8951 77,
8952 serde_json::to_vec(&ModuleHelloBody {
8953 manifest: squatter,
8954 protocol_ver: PROTOCOL_VERSION,
8955 control_ops: None,
8956 launch_nonce: None,
8957 })
8958 .expect("HELLO serializes"),
8959 )
8960 .expect("HELLO frame builds");
8961 let response = handler
8962 .handle_control(ConnectionId::new(77), frame)
8963 .expect("reserved claim receives a typed refusal");
8964 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8965 assert_eq!(
8966 registry
8967 .active_registration_count()
8968 .expect("registry reads"),
8969 0,
8970 "a reserved capability refusal must not leave a catalog entry"
8971 );
8972 }
8973
8974 #[test]
8975 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8976 for settlement in ["timeout", "inconclusive", "drop"] {
8977 let breakers = RouteBindBreakers::default();
8978 let RouteBindAdmission::Admitted {
8979 guard: mut old,
8980 probe: false,
8981 } = breakers.admit("prov")
8982 else {
8983 panic!("ordinary relay admitted")
8984 };
8985 let RouteBindAdmission::Admitted {
8986 guard: mut opener, ..
8987 } = breakers.admit("prov")
8988 else {
8989 panic!("second relay admitted")
8990 };
8991 assert!(
8992 !opener
8993 .record_timeout(1, Duration::ZERO)
8994 .unwrap()
8995 .reopened_after_probe
8996 );
8997 let RouteBindAdmission::Admitted {
8998 guard: mut probe,
8999 probe: true,
9000 } = breakers.admit("prov")
9001 else {
9002 panic!("one cooldown probe admitted")
9003 };
9004 match settlement {
9005 "timeout" => assert!(
9006 !old.record_timeout(1, Duration::ZERO)
9007 .unwrap()
9008 .reopened_after_probe
9009 ),
9010 "inconclusive" => old.record_inconclusive(),
9011 "drop" => drop(old),
9012 _ => unreachable!(),
9013 }
9014 assert!(
9015 matches!(
9016 breakers.admit("prov"),
9017 RouteBindAdmission::Refused {
9018 probe_in_flight: true,
9019 ..
9020 }
9021 ),
9022 "{settlement} of a pre-open relay cannot release the real probe"
9023 );
9024 assert!(
9025 probe
9026 .record_timeout(1, Duration::ZERO)
9027 .unwrap()
9028 .reopened_after_probe
9029 );
9030 assert!(matches!(
9031 breakers.admit("prov"),
9032 RouteBindAdmission::Admitted { probe: true, .. }
9033 ));
9034 }
9035 let breakers = RouteBindBreakers::default();
9036 let admit = || match breakers.admit("prov") {
9037 RouteBindAdmission::Admitted { guard, .. } => guard,
9038 _ => panic!("relay admitted"),
9039 };
9040 admit().record_timeout(1, Duration::ZERO);
9041 let mut old_probe = admit();
9042 breakers.reset_for_new_module_connection("prov");
9043 admit().record_timeout(1, Duration::ZERO);
9044 let _new_probe = admit();
9045 old_probe.record_inconclusive();
9046 assert!(matches!(
9047 breakers.admit("prov"),
9048 RouteBindAdmission::Refused {
9049 probe_in_flight: true,
9050 ..
9051 }
9052 ));
9053 }
9054
9055 #[tokio::test]
9056 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
9057 for candidate in [false, true] {
9058 let registry = Arc::new(Registry::default());
9059 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9060 [("vault".to_string(), true), ("squatter".to_string(), true)],
9061 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
9062 );
9063 let conn = ConnectionId::new(77);
9064 let (ctx, mut rx) = route_ctx(conn);
9065 let initial = capability_manifest("squatter", &[], &[]);
9066 if candidate {
9067 registry
9068 .register_candidate_with_control_ops(
9069 initial.clone(),
9070 PROTOCOL_VERSION,
9071 conn,
9072 module_baseline_control_ops(),
9073 )
9074 .unwrap();
9075 handler
9076 .forwarding
9077 .register_candidate_module_connection(
9078 conn,
9079 "squatter".to_string(),
9080 PROTOCOL_VERSION,
9081 manifest_concurrency(&initial),
9082 ctx.egress.clone(),
9083 )
9084 .unwrap();
9085 } else {
9086 hello_via_sink(
9087 &handler,
9088 &ctx,
9089 &mut rx,
9090 hello_frame_with_manifest(initial.clone(), 1),
9091 )
9092 .await;
9093 }
9094 let response = handler
9095 .handle_control_frame(
9096 &ctx,
9097 catalog_update_with_capabilities_frame(
9098 2,
9099 capability_manifest("squatter", &["credentials-provider/v1"], &[])
9100 .capabilities
9101 .unwrap(),
9102 ),
9103 )
9104 .await
9105 .unwrap();
9106 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
9107 assert_eq!(
9108 registry
9109 .get_module_by_connection(conn)
9110 .unwrap()
9111 .unwrap()
9112 .manifest,
9113 initial
9114 );
9115 }
9116 }
9117
9118 #[test]
9119 fn server_describe_surfaces_required_capability_verdict_fields() {
9120 let registry = Arc::new(Registry::default());
9121 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
9122 [
9123 ("consumer".to_string(), true),
9124 ("provider".to_string(), false),
9125 ],
9126 BTreeMap::new(),
9127 );
9128 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
9129 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
9130 provides: Vec::new(),
9131 requires: vec![subc_protocol::manifest::CapabilityRequirement {
9132 capability: "credentials-provider/v1".to_string(),
9133 need: subc_protocol::manifest::CapabilityNeed::Required,
9134 }],
9135 must_never_reach: Vec::new(),
9136 });
9137 let hello = Frame::build(
9138 FrameType::Hello,
9139 control_flags(),
9140 0,
9141 0,
9142 78,
9143 serde_json::to_vec(&ModuleHelloBody {
9144 manifest: consumer,
9145 protocol_ver: PROTOCOL_VERSION,
9146 control_ops: None,
9147 launch_nonce: None,
9148 })
9149 .expect("HELLO serializes"),
9150 )
9151 .expect("HELLO frame builds");
9152 handler
9153 .handle_control(ConnectionId::new(78), hello)
9154 .expect("consumer registers");
9155 let describe = Frame::build(
9156 FrameType::Request,
9157 control_flags(),
9158 0,
9159 0,
9160 79,
9161 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
9162 .expect("request serializes"),
9163 )
9164 .expect("describe frame builds");
9165 let response = handler
9166 .handle_server_describe(describe)
9167 .expect("server.describe succeeds");
9168 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
9169 let requirement = &rendered["capability_requirements"][0];
9170 assert_eq!(requirement["consumer"], "consumer");
9171 assert_eq!(requirement["verdict"], "never_provided");
9172 assert_eq!(requirement["episode_seq"], 1);
9173 assert_eq!(requirement["config_satisfiable"], false);
9174 assert_eq!(requirement["runtime_available"], false);
9175 assert!(requirement["detail"]
9176 .as_str()
9177 .expect("detail string")
9178 .contains("credentials-provider/v1"));
9179 }
9180
9181 #[test]
9182 fn catalog_list_omits_capabilities_for_legacy_manifest() {
9183 let registry = Arc::new(Registry::default());
9184 let handler = ControlHandler::new(Arc::clone(®istry));
9185 let hello = handler
9186 .handle_control(
9187 ConnectionId::new(101),
9188 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
9189 )
9190 .expect("legacy HELLO registers");
9191 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9192
9193 let request = Frame::build(
9194 FrameType::Request,
9195 control_flags(),
9196 0,
9197 0,
9198 102,
9199 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9200 .expect("catalog request serializes"),
9201 )
9202 .expect("catalog request frame builds");
9203 let response = handler
9204 .handle_catalog_list(request, None)
9205 .expect("catalog list succeeds");
9206 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9207 assert!(
9208 body["modules"][0].get("capabilities").is_none(),
9209 "legacy manifest must retain an absent capabilities field on catalog.list"
9210 );
9211 }
9212
9213 #[test]
9214 fn hello_ack_omits_storage_when_no_storage_config() {
9215 let registry = Arc::new(Registry::default());
9216 let handler = ControlHandler::new(Arc::clone(®istry));
9217 let responses = handler
9218 .handle_control(
9219 ConnectionId::new(1),
9220 hello_frame("aft", PROTOCOL_VERSION, 7),
9221 )
9222 .unwrap();
9223 let ack = parse_ack(&responses[0]);
9224 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
9225 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
9226 }
9227
9228 #[tokio::test]
9229 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
9230 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
9231 let registry = Arc::new(Registry::default());
9232 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
9233 let responses = handler
9234 .handle_control(
9235 ConnectionId::new(1),
9236 hello_frame("aft", PROTOCOL_VERSION, 7),
9237 )
9238 .unwrap();
9239 let ack = parse_ack(&responses[0]);
9240 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
9241
9242 let described = handler
9243 .handle_control_frame(
9244 &route_ctx(ConnectionId::new(2)).0,
9245 Frame::build(
9246 FrameType::Request,
9247 control_flags(),
9248 0,
9249 0,
9250 9,
9251 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9252 )
9253 .unwrap(),
9254 )
9255 .await
9256 .unwrap();
9257 let ClientControlResponse::ServerDescribe { machine_id, .. } =
9258 serde_json::from_slice(&described[0].body).unwrap()
9259 else {
9260 panic!("server.describe answered with another shape");
9261 };
9262 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
9263 }
9264
9265 #[test]
9266 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
9267 let registry = Arc::new(Registry::default());
9270 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
9271 crate::daemon_config::StorageConfig::Sqlite {
9272 data_home: std::path::PathBuf::from("/data"),
9273 },
9274 ));
9275
9276 let responses = handler
9277 .handle_control(
9278 ConnectionId::new(1),
9279 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
9280 )
9281 .unwrap();
9282 let ack = parse_ack(&responses[0]);
9283 assert_eq!(
9284 ack.storage,
9285 Some(serde_json::json!({
9286 "module_id": "alfonso-routing",
9287 "storage_namespace": "default",
9288 "isolation": { "kind": "module" },
9289 "backend": {
9290 "backend": "sqlite",
9291 "path": "/data/cortexkit/alfonso-routing/store.db"
9292 }
9293 })),
9294 "the delivered descriptor is the module's own sqlite store path"
9295 );
9296 }
9297
9298 #[test]
9299 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
9300 let registry = Arc::new(Registry::default());
9301 let handler = ControlHandler::new(Arc::clone(®istry));
9302 let conn = ConnectionId::new(1);
9303 let responses = handler
9304 .handle_control(
9305 conn,
9306 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9307 )
9308 .unwrap();
9309 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9310 let registration = registry.get_module("aft").unwrap().unwrap();
9311 assert_eq!(registration.control_ops, module_baseline_control_ops());
9312
9313 let frame =
9314 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
9315 assert!(handler
9316 .guard_module_control_op(&frame, "aft", "route.bind")
9317 .unwrap()
9318 .is_none());
9319 let error = handler
9320 .guard_module_control_op(&frame, "aft", "test.synthetic")
9321 .unwrap()
9322 .expect("synthetic ungranted op should be rejected");
9323 assert_eq!(error.header.ty, FrameType::Error);
9324 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
9325 }
9326
9327 #[test]
9328 fn hello_control_ops_some_adds_optional_grants() {
9329 let registry = Arc::new(Registry::default());
9330 let handler = ControlHandler::new(Arc::clone(®istry));
9331 handler
9332 .handle_control(
9333 ConnectionId::new(1),
9334 hello_frame_with_control_ops(
9335 "aft",
9336 PROTOCOL_VERSION,
9337 7,
9338 Some(vec![
9339 "future.synthetic".to_string(),
9340 "route.bind".to_string(),
9341 ]),
9342 ),
9343 )
9344 .unwrap();
9345 let registration = registry.get_module("aft").unwrap().unwrap();
9346 assert_eq!(
9347 registration.control_ops,
9348 vec![
9349 "route.bind".to_string(),
9350 "route.status".to_string(),
9351 "future.synthetic".to_string(),
9352 ]
9353 );
9354 let frame =
9355 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9356 assert!(handler
9357 .guard_module_control_op(&frame, "aft", "future.synthetic")
9358 .unwrap()
9359 .is_none());
9360 }
9361
9362 #[tokio::test]
9363 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9364 let registry = Arc::new(Registry::default());
9365 let forwarding = Arc::new(ForwardingTable::default());
9366 let handler =
9367 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9368 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9369 hello_via_sink(
9370 &handler,
9371 &module_ctx,
9372 &mut module_rx,
9373 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9374 )
9375 .await;
9376
9377 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9378 let responses = handler
9379 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9380 .await
9381 .unwrap();
9382 assert_eq!(responses.len(), 1);
9383 assert_eq!(responses[0].header.ty, FrameType::Error);
9384 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9385 assert!(module_rx.try_recv().is_err());
9386 }
9387
9388 #[tokio::test]
9389 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9390 let registry = Arc::new(Registry::default());
9391 let forwarding = Arc::new(ForwardingTable::default());
9392 let handler =
9393 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9394 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9395 hello_via_sink(
9396 &handler,
9397 &module_ctx,
9398 &mut module_rx,
9399 hello_frame_with_control_ops(
9400 "aft",
9401 PROTOCOL_VERSION,
9402 7,
9403 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9404 ),
9405 )
9406 .await;
9407
9408 let project_root = unique_project_root("demux");
9409 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9410 let route_handler = handler.clone();
9411 let route_task = tokio::spawn(async move {
9412 route_handler
9413 .handle_control_frame(
9414 &route_client_ctx,
9415 route_open_frame(100, "aft", project_root),
9416 )
9417 .await
9418 .unwrap()
9419 });
9420 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9421 .await
9422 .unwrap()
9423 .unwrap();
9424 assert!(matches!(
9425 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
9426 ModuleControlRequest::RouteBind { .. }
9427 ));
9428
9429 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
9430 let health_handler = handler.clone();
9431 let health_task = tokio::spawn(async move {
9432 health_handler
9433 .handle_control_frame(
9434 &health_client_ctx,
9435 supervisor_health_probe_frame(101, "aft"),
9436 )
9437 .await
9438 .unwrap()
9439 });
9440 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9441 .await
9442 .unwrap()
9443 .unwrap();
9444 assert_eq!(
9445 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9446 ModuleControlRequest::HealthCheck {}
9447 );
9448
9449 handler
9450 .handle_control_frame(
9451 &module_ctx,
9452 health_response(health_frame.header.corr, HealthStatus::Degraded),
9453 )
9454 .await
9455 .unwrap();
9456 let health_response = health_task.await.unwrap();
9457 assert_eq!(health_response.len(), 1);
9458 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9459 ClientControlResponse::SupervisorHealthProbe {
9460 module_id,
9461 status,
9462 detail,
9463 metrics,
9464 } => {
9465 assert_eq!(module_id, "aft");
9466 assert_eq!(status, HealthStatus::Degraded);
9467 assert_eq!(detail.as_deref(), Some("warming"));
9468 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
9469 }
9470 other => panic!("unexpected health response: {other:?}"),
9471 }
9472
9473 handler
9474 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9475 .await
9476 .unwrap();
9477 let route_response = route_task.await.unwrap();
9478 assert!(route_response.is_empty());
9479 let published = route_client_rx.recv().await.unwrap();
9480 assert!(matches!(
9481 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9482 ClientControlResponse::RouteOpen { .. }
9483 ));
9484 }
9485
9486 async fn relay_route_open(
9491 handler: &ControlHandler,
9492 client_connection: ConnectionId,
9493 client_egress: &FrameSink,
9494 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9495 corr: u64,
9496 module_id: &str,
9497 project_root_label: &str,
9498 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
9499 let ctx = RouteCtx {
9500 connection_id: client_connection,
9501 egress: client_egress.clone(),
9502 };
9503 let handler = handler.clone();
9504 let project_root = unique_project_root(project_root_label);
9505 let module_id = module_id.to_string();
9506 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
9507 let task = tokio::spawn(async move {
9508 let _guard = tracing::dispatcher::set_default(&dispatch);
9509 handler
9510 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
9511 .await
9512 .unwrap()
9513 });
9514 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
9515 .await
9516 .expect("module receives the relayed route.bind")
9517 .expect("module egress is open");
9518 (task, bind.frame)
9519 }
9520
9521 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
9522 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
9523 ModuleControlRequest::RouteBind {
9524 route_channel,
9525 epoch,
9526 ..
9527 } => (route_channel, epoch),
9528 other => panic!("expected a route.bind request, got {other:?}"),
9529 }
9530 }
9531
9532 fn published_route(frame: &Frame) -> (u16, u32) {
9533 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
9534 ClientControlResponse::RouteOpen {
9535 route_channel,
9536 route_epoch,
9537 } => (route_channel, route_epoch),
9538 other => panic!("expected a route.open response, got {other:?}"),
9539 }
9540 }
9541
9542 #[tokio::test]
9562 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
9563 let registry = Arc::new(Registry::default());
9564 let forwarding = Arc::new(ForwardingTable::default());
9565 let handler =
9566 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9567
9568 let module_connection = ConnectionId::new(30);
9569 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9570 hello_via_sink(
9571 &handler,
9572 &module_ctx,
9573 &mut module_rx,
9574 hello_frame("aft", PROTOCOL_VERSION, 7),
9575 )
9576 .await;
9577
9578 let dying_client = ConnectionId::new(31);
9579 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
9580
9581 let (first_task, first_bind) = relay_route_open(
9584 &handler,
9585 dying_client,
9586 &dying_ctx.egress,
9587 &mut module_rx,
9588 100,
9589 "aft",
9590 "closing-first",
9591 )
9592 .await;
9593 handler
9594 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
9595 .await
9596 .unwrap();
9597 assert!(first_task.await.unwrap().is_empty());
9598 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
9599
9600 let (second_task, second_bind) = relay_route_open(
9602 &handler,
9603 dying_client,
9604 &dying_ctx.egress,
9605 &mut module_rx,
9606 101,
9607 "aft",
9608 "closing-second",
9609 )
9610 .await;
9611 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9612
9613 assert!(forwarding
9616 .escalate_client_delivery_failure(
9617 dying_client,
9618 first_channel,
9619 first_epoch,
9620 CloseReason::new(
9621 "module_to_client_delivery_failed",
9622 "client egress refused a module frame",
9623 ),
9624 crate::forwarding::UndeliveredFrame {
9625 module_id: None,
9626 sink: &dying_ctx.egress,
9627 },
9628 )
9629 .unwrap());
9630 assert!(!dying_ctx.egress.is_closed());
9631
9632 let ack = handler
9634 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9635 .await;
9636 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9637 if module_loop_error.is_some() {
9638 handler.cleanup_connection(module_connection).unwrap();
9642 }
9643 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9647 .await
9648 .ok()
9649 .flatten();
9650
9651 assert!(
9653 registry
9654 .get_module_by_connection(module_connection)
9655 .unwrap()
9656 .is_some(),
9657 "one client's closing connection ended the shared module connection: \
9658 {module_loop_error:?}"
9659 );
9660 let cotenant = ConnectionId::new(32);
9662 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9663 let (cotenant_task, cotenant_bind) = relay_route_open(
9664 &handler,
9665 cotenant,
9666 &cotenant_ctx.egress,
9667 &mut module_rx,
9668 102,
9669 "aft",
9670 "closing-cotenant",
9671 )
9672 .await;
9673 handler
9674 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9675 .await
9676 .unwrap();
9677 assert!(cotenant_task.await.unwrap().is_empty());
9678 let (cotenant_channel, cotenant_epoch) =
9679 published_route(&cotenant_rx.recv().await.unwrap());
9680 assert!(matches!(
9681 forwarding
9682 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9683 .unwrap(),
9684 DataRoute::Client(DataRouteState::Bound(_))
9685 ));
9686
9687 let goodbye = post_ack_module_frame
9690 .expect("module receives a GOODBYE for the abandoned route channel");
9691 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9692 assert_eq!(goodbye.header.channel, abandoned_channel);
9693 assert_eq!(goodbye.header.epoch, abandoned_epoch);
9694
9695 assert!(dying_rx.try_recv().is_err());
9699 let second_response = second_task.await.unwrap();
9700 assert_eq!(second_response.len(), 1);
9701 assert_eq!(
9702 parse_error(&second_response[0])["code"],
9703 "target_unavailable"
9704 );
9705 }
9706
9707 async fn catalog_snapshot(
9710 handler: &ControlHandler,
9711 client_ctx: &RouteCtx,
9712 corr: u64,
9713 ) -> (u64, Vec<(String, bool)>) {
9714 let body =
9715 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
9716 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
9717 let reply = handler
9718 .handle_control_frame(client_ctx, request)
9719 .await
9720 .unwrap()
9721 .pop()
9722 .unwrap();
9723 let ClientControlResponse::CatalogList {
9724 generation,
9725 modules,
9726 ..
9727 } = serde_json::from_slice(&reply.body).unwrap()
9728 else {
9729 panic!("not a catalog.list reply");
9730 };
9731 (
9732 generation,
9733 modules
9734 .into_iter()
9735 .map(|entry| (entry.module_id, entry.ready))
9736 .collect(),
9737 )
9738 }
9739
9740 fn hello_held_before_forwarding() -> (
9746 ControlHandler,
9747 Arc<std::sync::Barrier>,
9748 std::thread::JoinHandle<Vec<Frame>>,
9749 RouteCtx,
9750 mpsc::Receiver<crate::router::OutboundFrame>,
9751 ) {
9752 let hold = Arc::new(std::sync::Barrier::new(2));
9753 let handler = ControlHandler::with_forwarding(
9754 Arc::new(Registry::default()),
9755 Arc::new(ForwardingTable::default()),
9756 )
9757 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
9758 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9759 let hello = {
9760 let handler = handler.clone();
9761 let module_ctx = module_ctx.clone();
9762 std::thread::spawn(move || {
9763 handler
9764 .handle_hello(
9765 module_ctx.connection_id,
9766 Some(module_ctx.egress.clone()),
9767 hello_frame("aft", PROTOCOL_VERSION, 7),
9768 )
9769 .unwrap()
9770 })
9771 };
9772 (handler, hold, hello, module_ctx, module_rx)
9773 }
9774
9775 #[tokio::test]
9784 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
9785 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
9786 let client = ConnectionId::new(41);
9787 let (client_ctx, mut client_rx) = route_ctx(client);
9788
9789 hold.wait();
9790 assert!(handler.registry.get_module("aft").unwrap().is_some());
9792 assert!(!handler
9793 .forwarding
9794 .has_live_module_connection("aft")
9795 .unwrap());
9796 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
9797 assert!(
9798 listed.is_empty(),
9799 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
9800 );
9801 let refused = handler
9802 .handle_control_frame(
9803 &client_ctx,
9804 route_open_frame(301, "aft", unique_project_root("listed-window")),
9805 )
9806 .await
9807 .unwrap();
9808 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
9809 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
9810 let duplicate = handler
9811 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
9812 .await
9813 .unwrap();
9814 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9815
9816 hold.wait();
9817 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9818 let ack = module_rx.try_recv().unwrap().frame;
9819 assert_eq!(ack.header.ty, FrameType::HelloAck);
9820 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9821 assert_eq!(listed, vec![("aft".to_string(), true)]);
9822
9823 let (open_task, bind) = relay_route_open(
9824 &handler,
9825 client,
9826 &client_ctx.egress,
9827 &mut module_rx,
9828 303,
9829 "aft",
9830 "listed-after",
9831 )
9832 .await;
9833 assert!(handler
9834 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9835 .await
9836 .unwrap()
9837 .is_empty());
9838 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
9839 published_route(&client_rx.try_recv().unwrap().frame);
9840 }
9841
9842 #[tokio::test]
9849 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
9850 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
9851 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9852 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9853
9854 hold.wait();
9855 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
9856 assert!(listed.is_empty());
9857 assert_eq!(
9858 during, before,
9859 "the generation moved while the module was not listed"
9860 );
9861
9862 hold.wait();
9863 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9864 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9865 assert_eq!(listed, vec![("aft".to_string(), true)]);
9866 assert_eq!(
9867 after,
9868 before + 1,
9869 "the module appeared without exactly one generation bump"
9870 );
9871 }
9872
9873 #[tokio::test]
9878 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
9879 let registry = Arc::new(Registry::default());
9880 let handler = ControlHandler::with_forwarding(
9881 Arc::clone(®istry),
9882 Arc::new(ForwardingTable::default()),
9883 );
9884 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9885 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9886 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9887 drop(module_rx);
9888
9889 let replies = handler
9890 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
9891 .await
9892 .unwrap();
9893 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
9894 assert!(registry.get_module("aft").unwrap().is_none());
9895 assert!(registry
9896 .get_module_by_connection(module_ctx.connection_id)
9897 .unwrap()
9898 .is_none());
9899 assert_eq!(
9900 catalog_snapshot(&handler, &client_ctx, 301).await,
9901 (before, Vec::new())
9902 );
9903 }
9904
9905 async fn route_open_held_after_bind_send() -> (
9909 ControlHandler,
9910 Arc<tokio::sync::Barrier>,
9911 RouteCtx,
9912 mpsc::Receiver<crate::router::OutboundFrame>,
9913 ) {
9914 let hold = Arc::new(tokio::sync::Barrier::new(2));
9915 let handler = ControlHandler::with_forwarding(
9916 Arc::new(Registry::default()),
9917 Arc::new(ForwardingTable::default()),
9918 )
9919 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
9920 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9921 hello_via_sink(
9922 &handler,
9923 &module_ctx,
9924 &mut module_rx,
9925 hello_frame("aft", PROTOCOL_VERSION, 7),
9926 )
9927 .await;
9928 (handler, hold, module_ctx, module_rx)
9929 }
9930
9931 #[tokio::test]
9938 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
9939 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9940 let client = ConnectionId::new(41);
9941 let (client_ctx, mut client_rx) = route_ctx(client);
9942
9943 let (open_task, bind) = relay_route_open(
9944 &handler,
9945 client,
9946 &client_ctx.egress,
9947 &mut module_rx,
9948 200,
9949 "aft",
9950 "acked-before-mark",
9951 )
9952 .await;
9953 let (module_channel, module_epoch) = route_bind_channel(&bind);
9954 hold.wait().await;
9956
9957 assert!(handler
9959 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9960 .await
9961 .unwrap()
9962 .is_empty());
9963 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
9964
9965 hold.wait().await;
9966 assert!(
9967 open_task.await.unwrap().is_empty(),
9968 "route.open answers accepted"
9969 );
9970
9971 match module_rx.try_recv() {
9974 Err(mpsc::error::TryRecvError::Empty) => {}
9975 Ok(outbound) => panic!(
9976 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
9977 outbound.frame.header.ty,
9978 outbound.frame.header.channel,
9979 outbound.frame.header.epoch
9980 ),
9981 Err(err) => panic!("module egress closed: {err}"),
9982 }
9983
9984 let router = Router::with_control_handler(Arc::new(handler.clone()));
9987 let request = Frame::build(
9988 FrameType::Request,
9989 Flags::new(true, Priority::Interactive, false),
9990 client_channel,
9991 client_epoch,
9992 201,
9993 b"first request".to_vec(),
9994 )
9995 .unwrap();
9996 router
9997 .route_for_connection(&client_ctx, request)
9998 .await
9999 .unwrap();
10000 let delivered = module_rx.try_recv().unwrap().frame;
10001 assert_eq!(delivered.header.ty, FrameType::Request);
10002 assert_eq!(
10003 (delivered.header.channel, delivered.header.epoch),
10004 (module_channel, module_epoch)
10005 );
10006 assert_eq!(delivered.body, b"first request");
10007 }
10008
10009 #[tokio::test]
10015 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
10016 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10017 let client = ConnectionId::new(42);
10018 let (client_ctx, _client_rx) = route_ctx(client);
10019
10020 let (open_task, bind) = relay_route_open(
10021 &handler,
10022 client,
10023 &client_ctx.egress,
10024 &mut module_rx,
10025 300,
10026 "aft",
10027 "released-before-mark",
10028 )
10029 .await;
10030 let (module_channel, module_epoch) = route_bind_channel(&bind);
10031 hold.wait().await;
10032
10033 handler.cleanup_connection(client).unwrap();
10034 assert!(
10035 module_rx.try_recv().is_err(),
10036 "client teardown must not GOODBYE a bind route.open is still queueing"
10037 );
10038
10039 hold.wait().await;
10040 let response = open_task.await.unwrap();
10041 assert_eq!(response.len(), 1);
10042 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10043
10044 let goodbye = module_rx
10045 .try_recv()
10046 .expect("module is told to drop the binding the daemon abandoned")
10047 .frame;
10048 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10049 assert_eq!(
10050 (goodbye.header.channel, goodbye.header.epoch),
10051 (module_channel, module_epoch)
10052 );
10053 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10054 }
10055
10056 #[tokio::test]
10061 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
10062 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
10063 let client = ConnectionId::new(43);
10064 let (client_ctx, client_rx) = route_ctx(client);
10065
10066 let (open_task, bind) = relay_route_open(
10067 &handler,
10068 client,
10069 &client_ctx.egress,
10070 &mut module_rx,
10071 400,
10072 "aft",
10073 "gone-client-before-mark",
10074 )
10075 .await;
10076 let (module_channel, module_epoch) = route_bind_channel(&bind);
10077 hold.wait().await;
10078
10079 drop(client_rx);
10080 assert!(client_ctx.egress.is_closed());
10081 handler
10082 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
10083 .await
10084 .unwrap();
10085
10086 hold.wait().await;
10087 let response = open_task.await.unwrap();
10088 assert_eq!(response.len(), 1);
10089 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
10090
10091 let goodbye = module_rx
10092 .try_recv()
10093 .expect("module is told to drop the binding it acked")
10094 .frame;
10095 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
10096 assert_eq!(
10097 (goodbye.header.channel, goodbye.header.epoch),
10098 (module_channel, module_epoch)
10099 );
10100 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
10101 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
10102 }
10103
10104 #[test]
10111 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
10112 let handler = ControlHandler::default();
10113 let module_connection = ConnectionId::new(30);
10114 let client_connection = ConnectionId::new(31);
10115
10116 handler
10117 .refuse_to_end_module_connection_for_a_client(
10118 module_connection,
10119 77,
10120 ForwardingError::ConnectionClosing {
10121 connection_id: client_connection,
10122 },
10123 )
10124 .expect("a closing client must never end the module connection");
10125
10126 assert!(matches!(
10127 handler.refuse_to_end_module_connection_for_a_client(
10128 module_connection,
10129 78,
10130 ForwardingError::ConnectionClosing {
10131 connection_id: module_connection,
10132 },
10133 ),
10134 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
10135 connection_id
10136 })) if connection_id == module_connection
10137 ));
10138 assert!(matches!(
10139 handler.refuse_to_end_module_connection_for_a_client(
10140 module_connection,
10141 79,
10142 ForwardingError::Poisoned,
10143 ),
10144 Err(RouterError::Forwarding(ForwardingError::Poisoned))
10145 ));
10146 assert!(matches!(
10147 handler.refuse_to_end_module_connection_for_a_client(
10148 module_connection,
10149 80,
10150 ForwardingError::StaleModuleEndpoint,
10151 ),
10152 Err(RouterError::Forwarding(
10153 ForwardingError::StaleModuleEndpoint
10154 ))
10155 ));
10156 }
10157
10158 #[tokio::test]
10171 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
10172 let registry = Arc::new(Registry::default());
10173 let forwarding = Arc::new(ForwardingTable::default());
10174 let supervisor = SupervisorHandle::new();
10175 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10176 let handler =
10177 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10178 .with_supervisor(supervisor);
10179
10180 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10181 hello_via_sink(
10182 &handler,
10183 &target_ctx,
10184 &mut target_rx,
10185 hello_frame("target", PROTOCOL_VERSION, 1),
10186 )
10187 .await;
10188
10189 let wrong_nonce = handler
10193 .handle_control_frame(
10194 &route_ctx(ConnectionId::new(91)).0,
10195 route_open_frame_with_admission_facts(
10196 20,
10197 "target",
10198 unique_project_root("admission-facts"),
10199 Some(subc_control::ConsumerIdentity {
10200 module_id: "fed".to_string(),
10201 launch_nonce: "not-the-real-nonce".to_string(),
10202 }),
10203 None,
10204 ),
10205 )
10206 .await
10207 .unwrap();
10208 assert_eq!(
10209 parse_error(&wrong_nonce[0])["code"],
10210 "bad_consumer_identity",
10211 "a mismatched launch nonce must be refused, not stamped Reserved"
10212 );
10213
10214 let never_spawned = handler
10218 .handle_control_frame(
10219 &route_ctx(ConnectionId::new(92)).0,
10220 route_open_frame_with_admission_facts(
10221 21,
10222 "target",
10223 unique_project_root("admission-facts"),
10224 Some(subc_control::ConsumerIdentity {
10225 module_id: "never-spawned".to_string(),
10226 launch_nonce: "any-nonce".to_string(),
10227 }),
10228 None,
10229 ),
10230 )
10231 .await
10232 .unwrap();
10233 assert_eq!(
10234 parse_error(&never_spawned[0])["code"],
10235 "bad_consumer_identity",
10236 "an unspawned module_id must be refused rather than accepted for lack of a record"
10237 );
10238 }
10239
10240 #[tokio::test]
10263 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
10264 let registry = Arc::new(Registry::default());
10265 let forwarding = Arc::new(ForwardingTable::default());
10266 let supervisor = SupervisorHandle::new();
10267 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10268 let handler =
10269 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10270 .with_supervisor(supervisor);
10271
10272 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
10273 hello_via_sink(
10274 &handler,
10275 &target_ctx,
10276 &mut target_rx,
10277 hello_frame("target", PROTOCOL_VERSION, 1),
10278 )
10279 .await;
10280
10281 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
10282 let route_handler = handler.clone();
10283 let route_task = tokio::spawn(async move {
10284 route_handler
10285 .handle_control_frame(
10286 &client_ctx,
10287 route_open_frame_with_admission_facts(
10288 30,
10289 "target",
10290 unique_project_root("admission-facts"),
10291 Some(subc_control::ConsumerIdentity {
10292 module_id: "fed".to_string(),
10293 launch_nonce: "fed-nonce".to_string(),
10294 }),
10295 None,
10296 ),
10297 )
10298 .await
10299 .unwrap()
10300 });
10301
10302 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10309 .await
10310 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
10311 .expect("module control channel closed before route.bind");
10312 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10313 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10314 panic!("expected route.bind")
10315 };
10316 assert_eq!(
10317 principal,
10318 Some(Principal::Reserved {
10319 module_id: "fed".to_string()
10320 }),
10321 "a correctly attested consumer must be stamped Reserved for its own id"
10322 );
10323
10324 handler
10325 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10326 .await
10327 .unwrap();
10328 assert!(route_task.await.unwrap().is_empty());
10329 assert!(
10330 matches!(
10331 serde_json::from_slice::<ClientControlResponse>(
10332 &client_rx.recv().await.unwrap().body
10333 )
10334 .unwrap(),
10335 ClientControlResponse::RouteOpen { .. }
10336 ),
10337 "the route must actually open, not merely avoid an error"
10338 );
10339 }
10340
10341 #[tokio::test(start_paused = true)]
10342 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10343 let registry = Arc::new(Registry::default());
10344 let forwarding = Arc::new(ForwardingTable::default());
10345 let supervisor = SupervisorHandle::new();
10346 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10347 let handler =
10348 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10349 .with_supervisor(supervisor);
10350
10351 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10352 hello_via_sink(
10353 &handler,
10354 &target_ctx,
10355 &mut target_rx,
10356 hello_frame("target", PROTOCOL_VERSION, 1),
10357 )
10358 .await;
10359
10360 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10361 let direct_handler = handler.clone();
10362 let direct_open = tokio::spawn(async move {
10363 direct_handler
10364 .handle_control_frame(
10365 &direct_ctx,
10366 route_open_frame(2, "target", unique_project_root("route-census-direct")),
10367 )
10368 .await
10369 .unwrap()
10370 });
10371 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10372 .await
10373 .expect("no direct route.bind within 5s")
10374 .expect("target control channel closed before direct route.bind");
10375 handler
10376 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10377 .await
10378 .unwrap();
10379 assert!(direct_open.await.unwrap().is_empty());
10380 let _ = direct_rx.recv().await.unwrap();
10381
10382 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10383 let reserved_handler = handler.clone();
10384 let reserved_open = tokio::spawn(async move {
10385 reserved_handler
10386 .handle_control_frame(
10387 &reserved_ctx,
10388 route_open_frame_with_admission_facts(
10389 3,
10390 "target",
10391 unique_project_root("admission-facts"),
10392 Some(ConsumerIdentity {
10393 module_id: "fed".to_string(),
10394 launch_nonce: "fed-nonce".to_string(),
10395 }),
10396 None,
10397 ),
10398 )
10399 .await
10400 .unwrap()
10401 });
10402 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10403 .await
10404 .expect("no reserved route.bind within 5s")
10405 .expect("target control channel closed before reserved route.bind");
10406 handler
10407 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10408 .await
10409 .unwrap();
10410 assert!(reserved_open.await.unwrap().is_empty());
10411 let _ = reserved_rx.recv().await.unwrap();
10412
10413 forwarding
10414 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10415 .unwrap();
10416 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
10417 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
10418 module_id: Some("target".to_string()),
10419 })
10420 .unwrap();
10421 let census_frame =
10422 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
10423 let response = handler
10424 .handle_control_frame(&census_ctx, census_frame)
10425 .await
10426 .unwrap()
10427 .pop()
10428 .unwrap();
10429 let actual: Value = serde_json::from_slice(&response.body).unwrap();
10430 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
10431 assert!(matches!(
10432 decoded,
10433 ClientControlResponse::SupervisorRoutes { .. }
10434 ));
10435 let routes = actual["modules"][0]["routes"].as_array().unwrap();
10436 assert_eq!(routes.len(), 2);
10437 assert!(routes.iter().all(|route| route["draining"] == true));
10438 assert!(
10441 routes.iter().all(|route| route["drain_reason"] == "reload"),
10442 "draining routes must name the drain's reason: {routes:?}"
10443 );
10444 assert!(routes.iter().any(|route| {
10445 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
10446 }));
10447 assert!(routes.iter().any(|route| {
10448 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
10449 }));
10450
10451 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10452 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
10453 if std::env::var_os("UPDATE_GOLDEN").is_some() {
10454 std::fs::write(
10455 &golden_path,
10456 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
10457 )
10458 .unwrap();
10459 }
10460 let expected: Value =
10461 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
10462 assert_eq!(actual, expected);
10463 }
10464
10465 async fn query_live_roots(
10466 handler: &ControlHandler,
10467 module_ctx: &RouteCtx,
10468 ) -> ModuleControlResponseToModule {
10469 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
10470 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
10471 let response = handler
10472 .handle_control_frame(module_ctx, frame)
10473 .await
10474 .unwrap()
10475 .pop()
10476 .unwrap();
10477 serde_json::from_slice(&response.body).unwrap()
10478 }
10479
10480 #[tokio::test(start_paused = true)]
10481 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
10482 let registry = Arc::new(Registry::default());
10483 let forwarding = Arc::new(ForwardingTable::default());
10484 let handler = ControlHandler::with_forwarding(registry, forwarding);
10485 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
10486 hello_via_sink(
10487 &handler,
10488 &target_ctx,
10489 &mut target_rx,
10490 hello_frame("target", PROTOCOL_VERSION, 1),
10491 )
10492 .await;
10493 let root = unique_project_root("live-roots-known");
10494 let path = ProjectRootId::from_path_allowing_missing(root.path())
10495 .unwrap()
10496 .as_path()
10497 .to_path_buf();
10498 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
10499 let open_handler = handler.clone();
10500 let opened = tokio::spawn(async move {
10501 open_handler
10502 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
10503 .await
10504 .unwrap()
10505 });
10506 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10507 .await
10508 .unwrap()
10509 .unwrap();
10510 handler
10511 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
10512 .await
10513 .unwrap();
10514 assert!(opened.await.unwrap().is_empty());
10515 let _ = client_rx.recv().await.unwrap();
10516
10517 let root = unique_project_root("live-roots-pending");
10518 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
10519 .unwrap()
10520 .as_path()
10521 .to_path_buf();
10522 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
10523 let open_handler = handler.clone();
10524 let pending = tokio::spawn(async move {
10525 open_handler
10526 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
10527 .await
10528 .unwrap()
10529 });
10530 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10531 .await
10532 .unwrap()
10533 .unwrap();
10534 let actual = query_live_roots(&handler, &target_ctx).await;
10535 let ModuleControlResponseToModule::LiveRoots {
10536 roots,
10537 unknown_root_bindings,
10538 total_bindings,
10539 } = actual
10540 else {
10541 panic!("expected live roots")
10542 };
10543 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
10544 assert_eq!(unknown_root_bindings, 0);
10545 assert_eq!(
10546 roots.len(),
10547 2,
10548 "root-known arm must retain each canonical root"
10549 );
10550 assert_eq!(
10551 total_bindings,
10552 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10553 );
10554 let counts = roots
10555 .iter()
10556 .map(|root| (root.project_root.clone(), root.bound, root.pending))
10557 .collect::<Vec<_>>();
10558 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
10559 expected.sort_by(|a, b| a.0.cmp(&b.0));
10560 assert_eq!(
10561 counts, expected,
10562 "roots must sort by path and count pending separately"
10563 );
10564 handler
10565 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
10566 .await
10567 .unwrap();
10568 assert!(pending.await.unwrap().is_empty());
10569 }
10570
10571 #[tokio::test(start_paused = true)]
10572 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
10573 let forwarding = Arc::new(ForwardingTable::default());
10574 let handler =
10575 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10576 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
10577 hello_via_sink(
10578 &handler,
10579 &target_ctx,
10580 &mut target_rx,
10581 hello_frame("target", PROTOCOL_VERSION, 1),
10582 )
10583 .await;
10584 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
10585 let pending = forwarding
10586 .begin_route_bind_relay_for_test(
10587 client_ctx.connection_id,
10588 client_ctx.egress.clone(),
10589 2,
10590 "target",
10591 )
10592 .unwrap();
10593 forwarding
10594 .complete_pending_relay(
10595 target_ctx.connection_id,
10596 pending.corr,
10597 RouteBindRelayOutcome::Accepted,
10598 )
10599 .unwrap();
10600 let actual = query_live_roots(&handler, &target_ctx).await;
10601 let ModuleControlResponseToModule::LiveRoots {
10602 roots,
10603 unknown_root_bindings,
10604 total_bindings,
10605 } = actual
10606 else {
10607 panic!("expected live roots")
10608 };
10609 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
10610 assert_eq!(
10611 unknown_root_bindings, 1,
10612 "unknown-root arm must not read as no bindings"
10613 );
10614 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
10615 assert_eq!(
10616 total_bindings,
10617 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10618 );
10619 }
10620
10621 #[tokio::test(start_paused = true)]
10629 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
10630 let forwarding = Arc::new(ForwardingTable::default());
10631 let handler =
10632 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10633 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
10634 let replies = handler
10635 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
10636 .await
10637 .unwrap();
10638 let queued_by_hello = module_rx.len();
10639
10640 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
10641 let open_handler = handler.clone();
10642 let open = tokio::spawn(async move {
10643 open_handler
10644 .handle_control_frame(
10645 &client_ctx,
10646 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
10647 )
10648 .await
10649 .unwrap()
10650 });
10651 let mut spins = 0;
10653 while module_rx.len() == queued_by_hello {
10654 spins += 1;
10655 assert!(spins < 10_000, "route.open never queued a route.bind");
10656 tokio::task::yield_now().await;
10657 }
10658
10659 for reply in replies {
10661 module_ctx.egress.send(reply).await.unwrap();
10662 }
10663
10664 let first = module_rx.recv().await.unwrap().frame;
10665 assert_eq!(
10666 first.header.ty,
10667 FrameType::HelloAck,
10668 "the first frame a registering module reads must be its HELLO_ACK"
10669 );
10670 assert_eq!(first.header.corr, 7);
10671 let second = module_rx.recv().await.unwrap().frame;
10672 assert_eq!(second.header.ty, FrameType::Request);
10673 assert!(
10674 matches!(
10675 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
10676 ModuleControlRequest::RouteBind { .. }
10677 ),
10678 "the route.bind follows the ack"
10679 );
10680 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
10681
10682 handler
10683 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
10684 .await
10685 .unwrap();
10686 assert!(open.await.unwrap().is_empty());
10687 let _ = client_rx.recv().await.unwrap();
10688 }
10689
10690 #[tokio::test(start_paused = true)]
10691 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
10692 let handler = ControlHandler::with_forwarding(
10693 Arc::new(Registry::default()),
10694 Arc::new(ForwardingTable::default()),
10695 );
10696 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
10697 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
10698 hello_via_sink(
10699 &handler,
10700 &first_ctx,
10701 &mut first_rx,
10702 hello_frame("first", PROTOCOL_VERSION, 1),
10703 )
10704 .await;
10705 hello_via_sink(
10706 &handler,
10707 &second_ctx,
10708 &mut second_rx,
10709 hello_frame("second", PROTOCOL_VERSION, 2),
10710 )
10711 .await;
10712 let root = unique_project_root("second-only");
10713 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
10714 let cloned = handler.clone();
10715 let open = tokio::spawn(async move {
10716 cloned
10717 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
10718 .await
10719 .unwrap()
10720 });
10721 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
10722 .await
10723 .unwrap()
10724 .unwrap();
10725 let first = query_live_roots(&handler, &first_ctx).await;
10726 let second = query_live_roots(&handler, &second_ctx).await;
10727 assert!(
10728 matches!(
10729 first,
10730 ModuleControlResponseToModule::LiveRoots {
10731 total_bindings: 0,
10732 ..
10733 }
10734 ),
10735 "cross-module scope must not expose another module's roots"
10736 );
10737 assert!(
10738 matches!(
10739 second,
10740 ModuleControlResponseToModule::LiveRoots {
10741 total_bindings: 1,
10742 ..
10743 }
10744 ),
10745 "second module must see its pending route"
10746 );
10747 handler
10748 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
10749 .await
10750 .unwrap();
10751 assert!(open.await.unwrap().is_empty());
10752 }
10753
10754 #[tokio::test(start_paused = true)]
10755 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
10756 let handler = ControlHandler::with_forwarding(
10757 Arc::new(Registry::default()),
10758 Arc::new(ForwardingTable::default()),
10759 );
10760 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
10761 hello_via_sink(
10762 &handler,
10763 &target_ctx,
10764 &mut target_rx,
10765 hello_frame("target", PROTOCOL_VERSION, 1),
10766 )
10767 .await;
10768 let actual = query_live_roots(&handler, &target_ctx).await;
10769 let ModuleControlResponseToModule::LiveRoots {
10770 roots,
10771 unknown_root_bindings,
10772 total_bindings,
10773 } = actual
10774 else {
10775 panic!("expected live roots")
10776 };
10777 assert!(roots.is_empty());
10778 assert_eq!(unknown_root_bindings, 0);
10779 assert_eq!(total_bindings, 0);
10780 assert_eq!(
10781 total_bindings,
10782 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10783 );
10784 }
10785
10786 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
10793 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10794 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
10795 let text = std::fs::read_to_string(&path)
10796 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
10797 let vectors: Vec<(String, Value)> = text
10798 .lines()
10799 .filter(|line| !line.trim().is_empty())
10800 .map(|line| {
10801 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
10802 let id = entry["corpus_id"]
10803 .as_str()
10804 .expect("every vector carries a corpus_id")
10805 .to_string();
10806 (id, entry["package"].clone())
10807 })
10808 .collect();
10809 assert_eq!(
10813 vectors.len(),
10814 3,
10815 "vendored fed corpus changed size; re-sync from subc-federation"
10816 );
10817
10818 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
10833 let richest = vectors
10834 .iter()
10835 .filter_map(|(_, package)| package.as_object())
10836 .map(|object| {
10837 object
10838 .keys()
10839 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
10840 .count()
10841 })
10842 .max()
10843 .unwrap_or(0);
10844 assert!(
10845 richest >= 2,
10846 "vendored corpus no longer carries a package with unmodelled fields, \
10847 so the relay test can no longer distinguish a verbatim relay from a lossy one"
10848 );
10849
10850 vectors
10851 }
10852
10853 #[tokio::test]
10861 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
10862 for (corpus_id, package) in fed_admission_facts_vectors() {
10863 let registry = Arc::new(Registry::default());
10864 let forwarding = Arc::new(ForwardingTable::default());
10865 let supervisor = SupervisorHandle::new();
10866 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10867 let handler =
10868 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10869 .with_supervisor(supervisor)
10870 .with_admission_facts_config(
10871 Some("fed".to_string()),
10872 Some(vec!["target".to_string()]),
10873 );
10874
10875 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10876 hello_via_sink(
10877 &handler,
10878 &target_ctx,
10879 &mut target_rx,
10880 hello_frame("target", PROTOCOL_VERSION, 1),
10881 )
10882 .await;
10883
10884 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
10885 let route_handler = handler.clone();
10886 let expected = package.clone();
10887 let route_task = tokio::spawn(async move {
10888 route_handler
10889 .handle_control_frame(
10890 &client_ctx,
10891 route_open_frame_with_admission_facts(
10892 20,
10893 "target",
10894 unique_project_root("admission-facts"),
10895 Some(subc_control::ConsumerIdentity {
10896 module_id: "fed".to_string(),
10897 launch_nonce: "fed-nonce".to_string(),
10898 }),
10899 Some(package),
10900 ),
10901 )
10902 .await
10903 .unwrap()
10904 });
10905
10906 let bind_frame = target_rx.recv().await.unwrap();
10907 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10908 let ModuleControlRequest::RouteBind {
10909 admission_facts, ..
10910 } = bind
10911 else {
10912 panic!("{corpus_id}: expected route.bind")
10913 };
10914 assert_eq!(
10915 admission_facts,
10916 Some(expected),
10917 "{corpus_id}: relay must not add, drop or reshape any field"
10918 );
10919
10920 handler
10921 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10922 .await
10923 .unwrap();
10924 route_task.await.unwrap();
10925 }
10926 }
10927
10928 #[tokio::test]
10929 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
10930 let registry = Arc::new(Registry::default());
10931 let forwarding = Arc::new(ForwardingTable::default());
10932 let supervisor = SupervisorHandle::new();
10933 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10934 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
10935 let handler =
10936 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10937 .with_supervisor(supervisor)
10938 .with_admission_facts_config(
10939 Some("fed".to_string()),
10940 Some(vec!["target".to_string()]),
10941 );
10942
10943 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
10944 hello_via_sink(
10945 &handler,
10946 &target_ctx,
10947 &mut target_rx,
10948 hello_frame("target", PROTOCOL_VERSION, 1),
10949 )
10950 .await;
10951 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
10952 hello_via_sink(
10953 &handler,
10954 &other_ctx,
10955 &mut other_rx,
10956 hello_frame("other", PROTOCOL_VERSION, 2),
10957 )
10958 .await;
10959
10960 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
10961 let expected_facts = facts.clone();
10962 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
10963 let route_handler = handler.clone();
10964 let route_task = tokio::spawn(async move {
10965 route_handler
10966 .handle_control_frame(
10967 &client_ctx,
10968 route_open_frame_with_admission_facts(
10969 10,
10970 "target",
10971 unique_project_root("admission-facts"),
10972 Some(subc_control::ConsumerIdentity {
10973 module_id: "fed".to_string(),
10974 launch_nonce: "fed-nonce".to_string(),
10975 }),
10976 Some(facts.clone()),
10977 ),
10978 )
10979 .await
10980 .unwrap()
10981 });
10982 let bind_frame = target_rx.recv().await.unwrap();
10983 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10984 let ModuleControlRequest::RouteBind {
10985 admission_facts, ..
10986 } = bind
10987 else {
10988 panic!("expected route.bind")
10989 };
10990 assert_eq!(admission_facts, Some(expected_facts));
10991 handler
10992 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10993 .await
10994 .unwrap();
10995 assert!(route_task.await.unwrap().is_empty());
10996 assert!(matches!(
10997 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
10998 .unwrap(),
10999 ClientControlResponse::RouteOpen { .. }
11000 ));
11001
11002 let direct = handler
11003 .handle_control_frame(
11004 &route_ctx(ConnectionId::new(73)).0,
11005 route_open_frame_with_admission_facts(
11006 11,
11007 "target",
11008 unique_project_root("admission-facts"),
11009 None,
11010 Some(json!({"x": 1})),
11011 ),
11012 )
11013 .await
11014 .unwrap();
11015 assert_eq!(
11016 parse_error(&direct[0])["code"],
11017 "admission_facts_not_permitted"
11018 );
11019
11020 let different_reserved = handler
11021 .handle_control_frame(
11022 &route_ctx(ConnectionId::new(77)).0,
11023 route_open_frame_with_admission_facts(
11024 15,
11025 "target",
11026 unique_project_root("admission-facts"),
11027 Some(subc_control::ConsumerIdentity {
11028 module_id: "other".to_string(),
11029 launch_nonce: "other-nonce".to_string(),
11030 }),
11031 Some(json!({"x": 1})),
11032 ),
11033 )
11034 .await
11035 .unwrap();
11036 assert_eq!(
11037 parse_error(&different_reserved[0])["code"],
11038 "admission_facts_not_permitted"
11039 );
11040
11041 let other_target = handler
11042 .handle_control_frame(
11043 &route_ctx(ConnectionId::new(74)).0,
11044 route_open_frame_with_admission_facts(
11045 12,
11046 "other",
11047 unique_project_root("admission-facts"),
11048 Some(subc_control::ConsumerIdentity {
11049 module_id: "fed".to_string(),
11050 launch_nonce: "fed-nonce".to_string(),
11051 }),
11052 Some(json!({"x": 1})),
11053 ),
11054 )
11055 .await
11056 .unwrap();
11057 assert_eq!(
11058 parse_error(&other_target[0])["code"],
11059 "admission_facts_target_not_allowed"
11060 );
11061
11062 let nonexistent = handler
11063 .handle_control_frame(
11064 &route_ctx(ConnectionId::new(75)).0,
11065 route_open_frame_with_admission_facts(
11066 13,
11067 "missing",
11068 unique_project_root("admission-facts"),
11069 None,
11070 Some(json!({"x": 1})),
11071 ),
11072 )
11073 .await
11074 .unwrap();
11075 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
11076
11077 let described = handler
11078 .handle_control_frame(
11079 &route_ctx(ConnectionId::new(76)).0,
11080 Frame::build(
11081 FrameType::Request,
11082 control_flags(),
11083 0,
11084 0,
11085 14,
11086 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11087 )
11088 .unwrap(),
11089 )
11090 .await
11091 .unwrap();
11092 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11093 serde_json::from_slice(&described[0].body).unwrap()
11094 else {
11095 panic!("expected server.describe response")
11096 };
11097 assert!(capabilities
11098 .iter()
11099 .any(|cap| cap == "admission_facts_relay_v1"));
11100 }
11101
11102 #[tokio::test]
11103 async fn admission_facts_without_configured_carrier_are_rejected() {
11104 let registry = Arc::new(Registry::default());
11105 let forwarding = Arc::new(ForwardingTable::default());
11106 let handler = ControlHandler::with_forwarding(registry, forwarding);
11107 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
11108 hello_via_sink(
11109 &handler,
11110 &target_ctx,
11111 &mut target_rx,
11112 hello_frame("target", PROTOCOL_VERSION, 1),
11113 )
11114 .await;
11115
11116 let responses = handler
11117 .handle_control_frame(
11118 &route_ctx(ConnectionId::new(79)).0,
11119 route_open_frame_with_admission_facts(
11120 16,
11121 "target",
11122 unique_project_root("admission-facts"),
11123 None,
11124 Some(json!({"x": 1})),
11125 ),
11126 )
11127 .await
11128 .unwrap();
11129 assert_eq!(
11130 parse_error(&responses[0])["code"],
11131 "admission_facts_not_permitted"
11132 );
11133 }
11134
11135 #[tokio::test]
11136 async fn route_open_relays_consumer_capabilities_verbatim() {
11137 let registry = Arc::new(Registry::default());
11138 let forwarding = Arc::new(ForwardingTable::default());
11139 let handler =
11140 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11141 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
11142 hello_via_sink(
11143 &handler,
11144 &module_ctx,
11145 &mut module_rx,
11146 hello_frame("aft", PROTOCOL_VERSION, 7),
11147 )
11148 .await;
11149
11150 let expected = vec!["elicitation".to_string(), "roots".to_string()];
11151 let expected_for_request = expected.clone();
11152 let project_root = unique_project_root("consumer-capabilities-present");
11153 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
11154 let route_handler = handler.clone();
11155 let route_task = tokio::spawn(async move {
11156 route_handler
11157 .handle_control_frame(
11158 &client_ctx,
11159 route_open_frame_with_consumer_capabilities(
11160 401,
11161 "aft",
11162 project_root,
11163 Some(expected_for_request),
11164 ),
11165 )
11166 .await
11167 .unwrap()
11168 });
11169 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11170 .await
11171 .unwrap()
11172 .unwrap();
11173 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11174 let ModuleControlRequest::RouteBind {
11175 consumer_capabilities,
11176 ..
11177 } = bind
11178 else {
11179 panic!("expected route.bind request, got {bind:?}");
11180 };
11181 assert_eq!(consumer_capabilities, Some(expected.clone()));
11182
11183 handler
11184 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11185 .await
11186 .unwrap();
11187 let route_response = route_task.await.unwrap();
11188 assert!(route_response.is_empty());
11189 let published = client_rx.recv().await.unwrap();
11190 assert!(matches!(
11191 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11192 ClientControlResponse::RouteOpen { .. }
11193 ));
11194 }
11195
11196 #[tokio::test]
11197 async fn route_open_without_consumer_capabilities_relays_none() {
11198 let registry = Arc::new(Registry::default());
11199 let forwarding = Arc::new(ForwardingTable::default());
11200 let handler =
11201 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11202 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
11203 hello_via_sink(
11204 &handler,
11205 &module_ctx,
11206 &mut module_rx,
11207 hello_frame("aft", PROTOCOL_VERSION, 7),
11208 )
11209 .await;
11210
11211 let project_root = unique_project_root("consumer-capabilities-absent");
11212 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
11213 let route_handler = handler.clone();
11214 let route_task = tokio::spawn(async move {
11215 route_handler
11216 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
11217 .await
11218 .unwrap()
11219 });
11220 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11221 .await
11222 .unwrap()
11223 .unwrap();
11224 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11225 let ModuleControlRequest::RouteBind {
11226 consumer_capabilities,
11227 ..
11228 } = bind
11229 else {
11230 panic!("expected route.bind request, got {bind:?}");
11231 };
11232 assert_eq!(consumer_capabilities, None);
11233
11234 handler
11235 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11236 .await
11237 .unwrap();
11238 let route_response = route_task.await.unwrap();
11239 assert!(route_response.is_empty());
11240 let published = client_rx.recv().await.unwrap();
11241 assert!(matches!(
11242 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11243 ClientControlResponse::RouteOpen { .. }
11244 ));
11245 }
11246
11247 async fn bind_role_versions_for(
11251 sent: Option<BTreeMap<String, String>>,
11252 connection: u64,
11253 ) -> Option<BTreeMap<String, String>> {
11254 let registry = Arc::new(Registry::default());
11255 let forwarding = Arc::new(ForwardingTable::default());
11256 let handler =
11257 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11258 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
11259 hello_via_sink(
11260 &handler,
11261 &module_ctx,
11262 &mut module_rx,
11263 hello_frame("aft", PROTOCOL_VERSION, 7),
11264 )
11265 .await;
11266 let project_root = unique_project_root("role-versions");
11267 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
11268 let route_handler = handler.clone();
11269 let route_task = tokio::spawn(async move {
11270 route_handler
11271 .handle_control_frame(
11272 &client_ctx,
11273 route_open_frame_with_role_versions(403, "aft", project_root, sent),
11274 )
11275 .await
11276 .unwrap()
11277 });
11278 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11279 .await
11280 .expect("a well-formed route.open reaches the module as a bind")
11281 .unwrap();
11282 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11283 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
11284 panic!("expected route.bind request, got {bind:?}");
11285 };
11286 handler
11287 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11288 .await
11289 .unwrap();
11290 assert!(route_task.await.unwrap().is_empty());
11291 let published = client_rx.recv().await.unwrap();
11292 assert!(matches!(
11293 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11294 ClientControlResponse::RouteOpen { .. }
11295 ));
11296 role_versions
11297 }
11298
11299 #[tokio::test]
11300 async fn route_open_relays_role_versions_verbatim() {
11301 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
11302 assert_eq!(
11303 bind_role_versions_for(Some(sent.clone()), 141).await,
11304 Some(sent)
11305 );
11306 }
11307
11308 #[tokio::test]
11311 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
11312 assert_eq!(bind_role_versions_for(None, 143).await, None);
11313 assert_eq!(
11314 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
11315 None
11316 );
11317 }
11318
11319 #[tokio::test]
11320 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
11321 let registry = Arc::new(Registry::default());
11322 let forwarding = Arc::new(ForwardingTable::default());
11323 let handler =
11324 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11325 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
11326 hello_via_sink(
11327 &handler,
11328 &module_ctx,
11329 &mut module_rx,
11330 hello_frame("aft", PROTOCOL_VERSION, 7),
11331 )
11332 .await;
11333
11334 let nine: BTreeMap<String, String> = (0..9)
11335 .map(|index| (format!("role-{index}"), "v1".to_string()))
11336 .collect();
11337 for (label, malformed) in [
11338 (
11339 "invalid role name",
11340 role_versions(&[("Tool_Provider", "v1")]),
11341 ),
11342 ("invalid version", role_versions(&[("tool-provider", "v0")])),
11343 ("nine entries", nine),
11344 ] {
11345 let responses = tokio::time::timeout(
11348 Duration::from_secs(1),
11349 handler.handle_control_frame(
11350 &route_ctx(ConnectionId::new(148)).0,
11351 route_open_frame_with_role_versions(
11352 404,
11353 "aft",
11354 unique_project_root("role-versions-malformed"),
11355 Some(malformed),
11356 ),
11357 ),
11358 )
11359 .await
11360 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11361 .unwrap();
11362 assert_eq!(responses.len(), 1, "{label}");
11363 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11364 let error = parse_error(&responses[0]);
11365 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11366 assert_eq!(
11367 error["detail"]["field"], "role_versions",
11368 "{label}: {error}"
11369 );
11370 assert!(
11371 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11372 "{label}: a malformed declaration is terminal"
11373 );
11374 assert!(
11375 module_rx.try_recv().is_err(),
11376 "{label}: the module must never see a bind"
11377 );
11378 }
11379 }
11380
11381 #[tokio::test]
11385 async fn route_role_versions_capability_is_advertised() {
11386 let handler = ControlHandler::new(Arc::new(Registry::default()));
11387 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11388 let ack = hello_via_sink(
11389 &handler,
11390 &ctx,
11391 &mut rx,
11392 hello_frame("m", PROTOCOL_VERSION, 1),
11393 )
11394 .await;
11395 let ack = parse_ack(&ack);
11396 assert!(
11397 ack.subc_capabilities
11398 .iter()
11399 .any(|c| c == "route-role-versions/v1"),
11400 "{:?}",
11401 ack.subc_capabilities
11402 );
11403
11404 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11405 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11406 let reply = handler
11407 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11408 .await
11409 .unwrap()
11410 .pop()
11411 .unwrap();
11412 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11413 serde_json::from_slice(&reply.body).unwrap()
11414 else {
11415 panic!("not a server.describe reply");
11416 };
11417 assert!(
11418 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
11419 "{capabilities:?}"
11420 );
11421 }
11422
11423 #[tokio::test]
11427 async fn catalog_events_capability_is_advertised() {
11428 let handler = ControlHandler::new(Arc::new(Registry::default()));
11429 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11430 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11431 let reply = handler
11432 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
11433 .await
11434 .unwrap()
11435 .pop()
11436 .unwrap();
11437 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11438 serde_json::from_slice(&reply.body).unwrap()
11439 else {
11440 panic!("not a server.describe reply");
11441 };
11442 assert!(
11443 capabilities
11444 .iter()
11445 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
11446 "{capabilities:?}"
11447 );
11448 }
11449
11450 #[tokio::test]
11451 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
11452 let handler = ControlHandler::new(Arc::new(Registry::default()));
11453 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
11454 let ack = hello_via_sink(
11455 &handler,
11456 &module,
11457 &mut module_rx,
11458 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
11459 )
11460 .await;
11461 let ack = parse_ack(&ack);
11462 assert!(
11463 ack.subc_capabilities
11464 .iter()
11465 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11466 "{:?}",
11467 ack.subc_capabilities
11468 );
11469 assert!(ack
11470 .subc_capabilities
11471 .iter()
11472 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11473
11474 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
11475 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11476 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11477 let reply = handler
11478 .handle_control_frame(&client, frame)
11479 .await
11480 .unwrap()
11481 .pop()
11482 .unwrap();
11483 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11484 serde_json::from_slice(&reply.body).unwrap()
11485 else {
11486 panic!("not a server.describe reply");
11487 };
11488 assert!(
11489 capabilities
11490 .iter()
11491 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11492 "{capabilities:?}"
11493 );
11494 assert!(capabilities
11495 .iter()
11496 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11497 }
11498
11499 #[tokio::test]
11500 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
11501 let handler = ControlHandler::new(Arc::new(Registry::default()));
11502 let (module, mut rx) = route_ctx(ConnectionId::new(156));
11503 let ack = hello_via_sink(
11504 &handler,
11505 &module,
11506 &mut rx,
11507 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
11508 )
11509 .await;
11510 assert!(parse_ack(&ack)
11511 .subc_capabilities
11512 .iter()
11513 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
11514 assert!(parse_ack(&ack)
11515 .subc_capabilities
11516 .iter()
11517 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
11518 let (client, _rx) = route_ctx(ConnectionId::new(157));
11519 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11520 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11521 let reply = handler
11522 .handle_control_frame(&client, frame)
11523 .await
11524 .unwrap()
11525 .pop()
11526 .unwrap();
11527 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11528 serde_json::from_slice(&reply.body).unwrap()
11529 else {
11530 panic!("server.describe reply")
11531 };
11532 assert!(capabilities
11533 .iter()
11534 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
11535 assert!(capabilities
11536 .iter()
11537 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
11538 }
11539
11540 #[tokio::test]
11541 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
11542 let handler = ControlHandler::new(Arc::new(Registry::default()));
11543 let (module, mut rx) = route_ctx(ConnectionId::new(154));
11544 let ack = hello_via_sink(
11545 &handler,
11546 &module,
11547 &mut rx,
11548 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
11549 )
11550 .await;
11551 assert!(parse_ack(&ack)
11552 .subc_capabilities
11553 .iter()
11554 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11555 let (client, _rx) = route_ctx(ConnectionId::new(155));
11556 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11557 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11558 let reply = handler
11559 .handle_control_frame(&client, frame)
11560 .await
11561 .unwrap()
11562 .pop()
11563 .unwrap();
11564 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11565 serde_json::from_slice(&reply.body).unwrap()
11566 else {
11567 panic!("not a server.describe reply")
11568 };
11569 assert!(capabilities
11570 .iter()
11571 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11572 }
11573
11574 #[tokio::test]
11575 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
11576 let registry = Arc::new(Registry::default());
11577 let forwarding = Arc::new(ForwardingTable::default());
11578 let handler =
11579 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11580 .with_health_probe_timeout(Duration::from_secs(5));
11581 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
11582 hello_via_sink(
11583 &handler,
11584 &module_ctx,
11585 &mut module_rx,
11586 non_routable_hello_frame_with_control_ops(
11587 "mcp",
11588 300,
11589 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11590 ),
11591 )
11592 .await;
11593 assert!(registry
11594 .get_module("mcp")
11595 .unwrap()
11596 .unwrap()
11597 .manifest
11598 .provides
11599 .is_empty());
11600
11601 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
11602 let route_response = handler
11603 .handle_control_frame(
11604 &route_client_ctx,
11605 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
11606 )
11607 .await
11608 .unwrap();
11609 assert_eq!(route_response[0].header.ty, FrameType::Error);
11610 assert_eq!(
11611 parse_error(&route_response[0])["code"],
11612 "target_unavailable"
11613 );
11614 assert!(parse_error(&route_response[0])["message"]
11615 .as_str()
11616 .unwrap()
11617 .contains("does not provide the requested target"));
11618 assert!(module_rx.try_recv().is_err());
11619
11620 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
11621 let health_handler = handler.clone();
11622 let health_task = tokio::spawn(async move {
11623 health_handler
11624 .handle_control_frame(
11625 &health_client_ctx,
11626 supervisor_health_probe_frame(302, "mcp"),
11627 )
11628 .await
11629 .unwrap()
11630 });
11631 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11632 .await
11633 .unwrap()
11634 .unwrap();
11635 assert_eq!(
11636 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11637 ModuleControlRequest::HealthCheck {}
11638 );
11639 handler
11640 .handle_control_frame(
11641 &module_ctx,
11642 health_response(health_frame.header.corr, HealthStatus::Ok),
11643 )
11644 .await
11645 .unwrap();
11646 let health_response = health_task.await.unwrap();
11647 assert_eq!(health_response[0].header.ty, FrameType::Response);
11648 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11649 ClientControlResponse::SupervisorHealthProbe {
11650 module_id, status, ..
11651 } => {
11652 assert_eq!(module_id, "mcp");
11653 assert_eq!(status, HealthStatus::Ok);
11654 }
11655 other => panic!("unexpected health response: {other:?}"),
11656 }
11657
11658 forwarding
11663 .cleanup_connection(module_ctx.connection_id)
11664 .unwrap();
11665 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
11666 let cleanup_response = tokio::time::timeout(
11667 Duration::from_millis(200),
11668 handler.handle_control_frame(
11669 &cleanup_probe_ctx,
11670 supervisor_health_probe_frame(303, "mcp"),
11671 ),
11672 )
11673 .await
11674 .expect("probe should fail immediately when the control lane is gone")
11675 .unwrap();
11676 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
11677 assert_eq!(
11678 parse_error(&cleanup_response[0])["code"],
11679 "target_unavailable"
11680 );
11681 assert!(parse_error(&cleanup_response[0])["message"]
11682 .as_str()
11683 .unwrap()
11684 .contains("no module connection"));
11685
11686 handler
11687 .cleanup_connection(module_ctx.connection_id)
11688 .unwrap();
11689 }
11690
11691 #[tokio::test]
11692 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
11693 let registry = Arc::new(Registry::default());
11694 let supervisor_handle = SupervisorHandle::new();
11695 let supervisor =
11696 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11697 .with_handle(supervisor_handle.clone())
11698 .with_connection_file_path(
11699 std::env::temp_dir()
11700 .join(format!("subc-route-open-warming-{}", std::process::id())),
11701 );
11702 let module = supervisor
11703 .supervise_configured(
11704 ModuleSpec {
11705 module_id: "warming".to_string(),
11706 program: fake_aft_stub_path(),
11707 args: Vec::new(),
11708 env: Vec::new(),
11709 reserved: false,
11710 reserved_prefixes: Vec::new(),
11711 protocol: ModuleProtocol::Subc,
11712 overlap: Default::default(),
11713 },
11714 true,
11715 )
11716 .unwrap();
11717 assert_eq!(module.state().unwrap(), ModuleState::Running);
11718
11719 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11720 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
11721 let response = handler
11722 .handle_control_frame(
11723 &ctx,
11724 route_open_frame(304, "warming", unique_project_root("warming")),
11725 )
11726 .await
11727 .unwrap();
11728 module.stop().await.unwrap();
11729
11730 assert_eq!(response[0].header.ty, FrameType::Error);
11731 let error = parse_error(&response[0]);
11732 assert_eq!(error["code"], "module_warming");
11733 assert!(error["message"]
11734 .as_str()
11735 .unwrap()
11736 .contains("state=running, enabled=true, live=false"));
11737 }
11738
11739 #[test]
11740 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
11741 let handler = ControlHandler::new(Arc::new(Registry::default()));
11742 let capture = EventCapture::default();
11743 let _subscriber =
11744 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11745 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
11746 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
11747 let pending = (0..limit).collect::<Vec<_>>();
11748 let response = handler
11749 .route_open_capacity_refusal(
11750 &ctx,
11751 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
11752 "busy",
11753 pending.len(),
11754 limit,
11755 )
11756 .unwrap();
11757 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11758 let event = capture
11759 .events()
11760 .into_iter()
11761 .find(|event| {
11762 event.target == "control"
11763 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
11764 })
11765 .expect("connection admission refusal event");
11766 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11767 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11768 }
11769
11770 #[test]
11771 fn route_open_target_cap_logs_admission_reason_and_capacity() {
11772 let handler = ControlHandler::new(Arc::new(Registry::default()));
11773 let capture = EventCapture::default();
11774 let _subscriber =
11775 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11776 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
11777 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
11778 let guards = (0..limit)
11779 .map(|_| {
11780 handler
11781 .route_bind_concurrency
11782 .try_admit("busy", limit)
11783 .unwrap()
11784 })
11785 .collect::<Vec<_>>();
11786 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
11787 Err(in_flight) => in_flight,
11788 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
11789 };
11790 let response = handler
11791 .route_open_target_capacity_refusal(
11792 &ctx,
11793 &route_open_frame(397, "busy", unique_project_root("target-cap")),
11794 "busy",
11795 in_flight,
11796 )
11797 .unwrap();
11798 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11799 let event = capture
11800 .events()
11801 .into_iter()
11802 .find(|event| {
11803 event.target == "control"
11804 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
11805 })
11806 .expect("target admission refusal event");
11807 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11808 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11809 drop(guards);
11810 }
11811
11812 #[tokio::test(flavor = "current_thread")]
11813 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
11814 let handler = ControlHandler::new(Arc::new(Registry::default()));
11815 let capture = EventCapture::default();
11816 let _subscriber =
11817 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11818 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
11819 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11820
11821 let _ = handler
11822 .handle_client_control_request(
11823 &ctx,
11824 frame,
11825 ClientControlRequest::SupervisorSetEnabled {
11826 module_id: "broca".to_string(),
11827 enabled: false,
11828 },
11829 &mut None,
11830 )
11831 .await
11832 .unwrap();
11833
11834 let events = capture
11835 .events()
11836 .into_iter()
11837 .filter(|event| {
11838 event.target == "control"
11839 && event.fields.get("message").map(String::as_str)
11840 == Some("supervisor request received")
11841 })
11842 .collect::<Vec<_>>();
11843 assert_eq!(events.len(), 1, "one supervisor request log line");
11844 let event = &events[0];
11845 assert_eq!(event.level, tracing::Level::INFO);
11846 assert_eq!(
11847 event.fields.get("op"),
11848 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
11849 );
11850 assert_eq!(
11851 event.fields.get("module_id"),
11852 Some(&"\"broca\"".to_string())
11853 );
11854 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
11855 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
11856 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
11857 }
11858
11859 #[tokio::test(flavor = "current_thread")]
11860 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
11861 let registry = Arc::new(Registry::default());
11862 let module_connection = ConnectionId::new(303);
11863 registry
11864 .register_with_control_ops(
11865 manifest("aft", PROTOCOL_VERSION),
11866 PROTOCOL_VERSION,
11867 module_connection,
11868 module_baseline_control_ops(),
11869 )
11870 .unwrap();
11871 let handler = ControlHandler::new(registry);
11872 let capture = EventCapture::default();
11873 let _subscriber =
11874 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11875 let (ctx, _rx) = route_ctx(module_connection);
11876 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11877
11878 let _ = handler
11879 .handle_client_control_request(
11880 &ctx,
11881 frame,
11882 ClientControlRequest::SupervisorSetEnabled {
11883 module_id: "broca".to_string(),
11884 enabled: false,
11885 },
11886 &mut None,
11887 )
11888 .await
11889 .unwrap();
11890
11891 let callers = capture
11892 .events()
11893 .into_iter()
11894 .filter(|event| {
11895 event.fields.get("message").map(String::as_str)
11896 == Some("supervisor request received")
11897 })
11898 .map(|event| event.fields.get("caller").cloned())
11899 .collect::<Vec<_>>();
11900 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
11901 }
11902
11903 #[tokio::test(flavor = "current_thread")]
11904 async fn supervisor_list_does_not_log_request_received() {
11905 let handler = ControlHandler::new(Arc::new(Registry::default()));
11906 let capture = EventCapture::default();
11907 let _subscriber =
11908 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11909 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
11910 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
11911
11912 handler
11913 .handle_client_control_request(
11914 &ctx,
11915 frame,
11916 ClientControlRequest::SupervisorList {},
11917 &mut None,
11918 )
11919 .await
11920 .unwrap();
11921
11922 assert!(capture.events().into_iter().all(|event| {
11923 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11924 }));
11925 }
11926
11927 #[tokio::test(flavor = "current_thread")]
11928 async fn supervisor_rescan_preview_does_not_log_request_received() {
11929 let handler = ControlHandler::new(Arc::new(Registry::default()));
11930 let capture = EventCapture::default();
11931 let _subscriber =
11932 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11933 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
11934 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
11935
11936 handler
11937 .handle_client_control_request(
11938 &ctx,
11939 frame,
11940 ClientControlRequest::SupervisorRescan { preview: true },
11941 &mut None,
11942 )
11943 .await
11944 .unwrap();
11945
11946 assert!(capture.events().into_iter().all(|event| {
11947 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11948 }));
11949 }
11950
11951 struct ApproveOperator;
11952 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
11953 fn prompt(
11954 &self,
11955 _: &str,
11956 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
11957 ) -> crate::operator_confirm::ProviderResult {
11958 crate::operator_confirm::ProviderResult::Approved
11959 }
11960 }
11961
11962 #[tokio::test]
11963 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
11964 assert_eq!(
11965 ControlHandler::default().privileged_operator_prompt,
11966 PrivilegedOperatorPrompt::Log
11967 );
11968 for (initial, changed, running) in [
11969 ("log", "enforce", PrivilegedOperatorPrompt::Log),
11970 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
11971 ] {
11972 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
11973 let path = dir.join("subc.jsonc");
11974 std::fs::write(
11975 &path,
11976 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
11977 )
11978 .unwrap();
11979 let config = crate::daemon_config::load(&path).unwrap().unwrap();
11980 let registry = Arc::new(Registry::default());
11981 let handle = SupervisorHandle::new();
11982 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
11983 .with_handle(handle.clone());
11984 let handler = ControlHandler::new(registry)
11985 .with_supervisor(handle)
11986 .with_privileged_operator_prompt(config.privileged_operator_prompt)
11987 .with_supervisor_rescan(supervisor, &path, config.port);
11988 handler.forwarding.operator_confirms().configure(
11989 Arc::new(ApproveOperator),
11990 crate::operator_confirm::OperatorLimits::default(),
11991 );
11992 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
11993
11994 for (disk_mode, needs_restart) in [
11995 (Some(initial), false),
11996 (Some(changed), true),
11997 (Some(changed), true),
11998 (None, running == PrivilegedOperatorPrompt::Enforce),
11999 (Some(initial), false),
12000 ] {
12001 let mut doc = json!({"version":1});
12002 if let Some(mode) = disk_mode {
12003 doc["privileged_operator_prompt"] = json!(mode);
12004 }
12005 std::fs::write(&path, doc.to_string()).unwrap();
12006 for preview in [true, false] {
12007 let request = ClientControlRequest::SupervisorRescan { preview };
12008 let frame = Frame::build(
12009 FrameType::Request,
12010 control_flags(),
12011 0,
12012 0,
12013 1,
12014 serde_json::to_vec(&request).unwrap(),
12015 )
12016 .unwrap();
12017 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12018 assert_eq!(frames.len(), 1);
12019 let ClientControlResponse::SupervisorRescan { result } =
12020 serde_json::from_slice(&frames[0].body).unwrap()
12021 else {
12022 panic!("expected a rescan response");
12023 };
12024 let expected = if needs_restart {
12025 vec!["privileged_operator_prompt"]
12026 } else {
12027 Vec::new()
12028 };
12029 assert_eq!(result.restart_required, expected);
12030 assert_eq!(handler.privileged_operator_prompt, running);
12031 assert_eq!(
12032 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
12033 running
12034 );
12035 }
12036 }
12037 }
12038 }
12039
12040 #[tokio::test]
12041 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
12042 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
12043 let path = dir.join("subc.jsonc");
12044 let registry = Arc::new(Registry::default());
12045 let handle = SupervisorHandle::new();
12046 let supervisor =
12047 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
12048 let handler = ControlHandler::new(registry)
12049 .with_supervisor(handle)
12050 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
12051 .with_supervisor_rescan(supervisor, &path, None);
12052 handler.forwarding.operator_confirms().configure(
12053 Arc::new(ApproveOperator),
12054 crate::operator_confirm::OperatorLimits::default(),
12055 );
12056 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
12057 for preview in [true, false] {
12058 std::fs::write(
12059 &path,
12060 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
12061 )
12062 .unwrap();
12063 let request = ClientControlRequest::SupervisorRescan { preview };
12064 let frame = Frame::build(
12065 FrameType::Request,
12066 control_flags(),
12067 0,
12068 0,
12069 1,
12070 serde_json::to_vec(&request).unwrap(),
12071 )
12072 .unwrap();
12073 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
12074 assert_eq!(frames.len(), 1);
12075 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
12076 assert_eq!(
12077 handler.privileged_operator_prompt,
12078 PrivilegedOperatorPrompt::Enforce
12079 );
12080 }
12081 }
12082
12083 #[tokio::test]
12088 async fn route_open_refusal_names_the_check_that_refused() {
12089 let handler = ControlHandler::new(Arc::new(Registry::default()));
12090 let capture = EventCapture::default();
12091 let _subscriber =
12092 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12093 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12094 let response = handler
12095 .handle_control_frame(
12096 &ctx,
12097 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
12098 )
12099 .await
12100 .unwrap();
12101
12102 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12103 let event = capture
12104 .events()
12105 .into_iter()
12106 .find(|event| {
12107 event.target == "control"
12108 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12109 })
12110 .expect("route.open refusal event");
12111 assert_eq!(
12112 event.fields.get("reason"),
12113 Some(&"\"not_registered\"".to_string())
12114 );
12115 }
12116
12117 #[tokio::test]
12118 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
12119 let registry = Arc::new(Registry::default());
12120 let supervisor_handle = SupervisorHandle::new();
12121 let supervisor =
12122 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12123 .with_handle(supervisor_handle.clone())
12124 .with_connection_file_path(std::env::temp_dir().join(format!(
12125 "subc-route-open-refusal-info-{}",
12126 std::process::id()
12127 )));
12128 let module = supervisor
12129 .supervise_configured(
12130 ModuleSpec {
12131 module_id: "warming".to_string(),
12132 program: fake_aft_stub_path(),
12133 args: Vec::new(),
12134 env: Vec::new(),
12135 reserved: false,
12136 reserved_prefixes: Vec::new(),
12137 protocol: ModuleProtocol::Subc,
12138 overlap: Default::default(),
12139 },
12140 true,
12141 )
12142 .unwrap();
12143 assert_eq!(module.state().unwrap(), ModuleState::Running);
12144
12145 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12146 assert!(handler
12147 .counters()
12148 .snapshot()
12149 .get("route_open_refused_by_code")
12150 .is_none());
12151 let capture = EventCapture::default();
12152 let _subscriber =
12153 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12154 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
12155 let response = handler
12156 .handle_control_frame(
12157 &ctx,
12158 route_open_frame(394, "warming", unique_project_root("refusal-info")),
12159 )
12160 .await
12161 .unwrap();
12162 module.stop().await.unwrap();
12163
12164 assert_eq!(parse_error(&response[0])["code"], "module_warming");
12165 let event = capture
12166 .events()
12167 .into_iter()
12168 .find(|event| {
12169 event.target == "control"
12170 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
12171 })
12172 .expect("route.open refusal event");
12173 assert_eq!(
12174 event.fields.get("module_id"),
12175 Some(&"\"warming\"".to_string())
12176 );
12177 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
12178 assert_eq!(
12179 event.fields.get("reason"),
12180 Some(&"\"supervised_not_registered\"".to_string())
12181 );
12182 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
12183 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
12184 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
12185 assert_eq!(
12186 handler.counters().snapshot()["route_open_refused_by_code"],
12187 json!({ "module_warming": 1 })
12188 );
12189 }
12190
12191 const OUTAGE_START: &str = "route.open refusing module: not serving";
12192 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
12193
12194 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
12195 capture
12196 .events()
12197 .into_iter()
12198 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
12199 .collect()
12200 }
12201
12202 fn supervise_stub(
12203 registry: &Arc<Registry>,
12204 module_id: &str,
12205 enabled: bool,
12206 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
12207 let supervisor_handle = SupervisorHandle::new();
12208 let supervisor =
12209 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
12210 .with_handle(supervisor_handle.clone())
12211 .with_connection_file_path(std::env::temp_dir().join(format!(
12212 "subc-route-outage-{module_id}-{}",
12213 std::process::id()
12214 )));
12215 let module = supervisor
12216 .supervise_configured(
12217 ModuleSpec {
12218 module_id: module_id.to_string(),
12219 program: fake_aft_stub_path(),
12220 args: Vec::new(),
12221 env: Vec::new(),
12222 reserved: false,
12223 reserved_prefixes: Vec::new(),
12224 protocol: ModuleProtocol::Subc,
12225 overlap: Default::default(),
12226 },
12227 enabled,
12228 )
12229 .unwrap();
12230 (supervisor_handle, module)
12231 }
12232
12233 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
12234 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
12235 module_id: module_id.to_string(),
12236 drain_timeout_ms: Some(50),
12237 })
12238 .unwrap();
12239 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
12240 }
12241
12242 #[test]
12246 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
12247 let registry = Arc::new(Registry::default());
12248 let forwarding = Arc::new(ForwardingTable::default());
12249 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12250 let second = ControlHandler::with_forwarding(registry, forwarding);
12251 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
12252 }
12253
12254 #[tokio::test(flavor = "current_thread")]
12258 async fn route_open_unknown_module_refusals_add_no_outage_state() {
12259 let handler = ControlHandler::new(Arc::new(Registry::default()));
12260 let capture = EventCapture::default();
12261 let _subscriber =
12262 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12263 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
12264 for corr in 0..8 {
12265 let response = handler
12266 .handle_control_frame(
12267 &ctx,
12268 route_open_frame(
12269 380 + corr,
12270 &format!("nobody-{corr}"),
12271 unique_project_root("outage-unknown"),
12272 ),
12273 )
12274 .await
12275 .unwrap();
12276 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12277 }
12278
12279 assert_eq!(handler.route_outages.tracked_module_count(), 0);
12280 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
12281 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
12282 }
12283
12284 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12289 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
12290 let registry = Arc::new(Registry::default());
12291 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
12292 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12293 let capture = EventCapture::default();
12294 let _subscriber =
12295 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12296 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
12297 handler.route_outages.record_accepted("outage-restart");
12300
12301 let response = handler
12302 .handle_control_frame(
12303 &ctx,
12304 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
12305 )
12306 .await
12307 .unwrap();
12308 assert_eq!(response[0].header.ty, FrameType::Error);
12309 let starts = outage_lines(&capture, OUTAGE_START);
12310 assert_eq!(starts.len(), 1, "{starts:?}");
12311 assert_eq!(starts[0].level, tracing::Level::WARN);
12312 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
12313 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
12314 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
12315 handler.route_outages.record_accepted("outage-restart");
12316 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
12317
12318 let restart = handler
12319 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
12320 .await
12321 .unwrap();
12322 assert_eq!(
12323 restart[0].header.ty,
12324 FrameType::Response,
12325 "{:?}",
12326 parse_error(&restart[0])
12327 );
12328 handler
12329 .handle_control_frame(
12330 &ctx,
12331 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
12332 )
12333 .await
12334 .unwrap();
12335 module.stop().await.unwrap();
12336
12337 let starts = outage_lines(&capture, OUTAGE_START);
12338 assert_eq!(starts.len(), 2, "{starts:?}");
12339 assert_eq!(starts[1].level, tracing::Level::INFO);
12340 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
12341 }
12342
12343 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12347 async fn failed_operator_restart_leaves_no_operator_mark() {
12348 let registry = Arc::new(Registry::default());
12349 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
12350 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12351 let capture = EventCapture::default();
12352 let _subscriber =
12353 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12354 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
12355 handler.route_outages.record_accepted("outage-disabled");
12356
12357 let restart = handler
12358 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
12359 .await
12360 .unwrap();
12361 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
12362 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
12363
12364 handler
12365 .handle_control_frame(
12366 &ctx,
12367 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
12368 )
12369 .await
12370 .unwrap();
12371 let starts = outage_lines(&capture, OUTAGE_START);
12372 assert_eq!(starts.len(), 1, "{starts:?}");
12373 assert_eq!(starts[0].level, tracing::Level::WARN);
12374 }
12375
12376 #[tokio::test(flavor = "current_thread")]
12377 async fn route_open_unknown_module_escapes_target_module_id() {
12378 let handler = ControlHandler::new(Arc::new(Registry::default()));
12379 let capture = EventCapture::default();
12380 let _subscriber =
12381 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12382 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
12383 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12384 let response = handler
12385 .handle_control_frame(
12386 &ctx,
12387 route_open_frame(
12388 395,
12389 hostile_module_id,
12390 unique_project_root("hostile-target-module-id"),
12391 ),
12392 )
12393 .await
12394 .unwrap();
12395
12396 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12397 let event = capture
12398 .events()
12399 .into_iter()
12400 .find(|event| {
12401 event.target == "control"
12402 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12403 })
12404 .expect("route.open unknown-module refusal event");
12405 let logged = event.fields.get("module_id").expect("module_id field");
12406 assert!(!logged.bytes().any(|byte| byte < 0x20));
12407 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12408 }
12409
12410 #[tokio::test(flavor = "current_thread")]
12411 async fn route_open_module_rejection_uses_daemon_counter_key() {
12412 let registry = Arc::new(Registry::default());
12413 let forwarding = Arc::new(ForwardingTable::default());
12414 let handler =
12415 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12416 let module_connection = ConnectionId::new(95);
12417 let (module_ctx, mut module_rx) = route_ctx(module_connection);
12418 hello_via_sink(
12419 &handler,
12420 &module_ctx,
12421 &mut module_rx,
12422 hello_frame("aft", PROTOCOL_VERSION, 395),
12423 )
12424 .await;
12425
12426 let client_connection = ConnectionId::new(96);
12427 let (client_ctx, _client_rx) = route_ctx(client_connection);
12428 let capture = EventCapture::default();
12429 let _subscriber =
12430 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12431 let (route_task, bind) = relay_route_open(
12432 &handler,
12433 client_connection,
12434 &client_ctx.egress,
12435 &mut module_rx,
12436 396,
12437 "aft",
12438 "hostile-module-code",
12439 )
12440 .await;
12441 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
12442 let rejection = Frame::build(
12443 FrameType::Error,
12444 control_flags(),
12445 0,
12446 0,
12447 bind.header.corr,
12448 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
12449 )
12450 .unwrap();
12451 handler
12452 .handle_control_frame(&module_ctx, rejection)
12453 .await
12454 .unwrap();
12455
12456 let response = route_task.await.unwrap();
12457 assert_eq!(parse_error(&response[0])["code"], hostile_code);
12458 let counters = handler.counters().snapshot();
12459 assert_eq!(
12460 counters["route_open_refused_by_code"],
12461 json!({ "module_rejected": 1 })
12462 );
12463 assert!(counters["route_open_refused_by_code"]
12464 .get(hostile_code)
12465 .is_none());
12466
12467 let event = capture
12468 .events()
12469 .into_iter()
12470 .find(|event| {
12471 event.target == "control"
12472 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
12473 })
12474 .expect("route.open module-rejection refusal event");
12475 let logged = event.fields.get("module_code").expect("module_code field");
12476 assert!(!logged.bytes().any(|byte| byte < 0x20));
12477 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12478 }
12479
12480 #[tokio::test]
12481 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
12482 let registry = Arc::new(Registry::default());
12483 let supervisor_handle = SupervisorHandle::new();
12484 let missing_program = std::env::temp_dir().join(format!(
12485 "subc-route-open-missing-program-{}",
12486 std::process::id()
12487 ));
12488 let supervisor =
12489 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12490 .with_handle(supervisor_handle.clone());
12491 let module = supervisor
12492 .supervise_configured(
12493 ModuleSpec {
12494 module_id: "failed".to_string(),
12495 program: missing_program,
12496 args: Vec::new(),
12497 env: Vec::new(),
12498 reserved: false,
12499 reserved_prefixes: Vec::new(),
12500 protocol: ModuleProtocol::Subc,
12501 overlap: Default::default(),
12502 },
12503 true,
12504 )
12505 .unwrap();
12506 assert_eq!(module.state().unwrap(), ModuleState::Failed);
12507
12508 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12509 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
12510 let response = handler
12511 .handle_control_frame(
12512 &ctx,
12513 route_open_frame(305, "failed", unique_project_root("failed")),
12514 )
12515 .await
12516 .unwrap();
12517
12518 assert_eq!(response[0].header.ty, FrameType::Error);
12519 let error = parse_error(&response[0]);
12520 assert_eq!(error["code"], "target_unavailable");
12521 assert!(error["message"]
12522 .as_str()
12523 .unwrap()
12524 .contains("state=failed, enabled=true, live=false"));
12525 }
12526
12527 #[tokio::test]
12528 async fn route_open_role_mismatch_remains_target_unavailable() {
12529 let registry = Arc::new(Registry::default());
12530 let handler = ControlHandler::new(Arc::clone(®istry));
12531 handler
12532 .handle_control(
12533 ConnectionId::new(41),
12534 non_routable_hello_frame_with_control_ops("health-only", 306, None),
12535 )
12536 .unwrap();
12537
12538 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
12539 let response = handler
12540 .handle_control_frame(
12541 &ctx,
12542 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
12543 )
12544 .await
12545 .unwrap();
12546
12547 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12548 assert!(parse_error(&response[0])["message"]
12549 .as_str()
12550 .unwrap()
12551 .contains("does not provide the requested target"));
12552 }
12553
12554 #[tokio::test]
12555 async fn route_open_inactive_registration_remains_target_unavailable() {
12556 let registry = Arc::new(Registry::default());
12557 let handler = ControlHandler::new(Arc::clone(®istry));
12558 handler
12559 .handle_control(
12560 ConnectionId::new(43),
12561 hello_frame("inactive", PROTOCOL_VERSION, 308),
12562 )
12563 .unwrap();
12564 assert!(registry
12565 .set_module_state_for_test("inactive", ChannelState::Closed)
12566 .unwrap());
12567
12568 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
12569 let response = handler
12570 .handle_control_frame(
12571 &ctx,
12572 route_open_frame(309, "inactive", unique_project_root("inactive")),
12573 )
12574 .await
12575 .unwrap();
12576
12577 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12578 assert!(parse_error(&response[0])["message"]
12579 .as_str()
12580 .unwrap()
12581 .contains("is not active"));
12582 }
12583
12584 #[tokio::test]
12585 async fn late_health_reply_is_recorded_through_the_module_response_path() {
12586 let registry = Arc::new(Registry::default());
12587 let forwarding = Arc::new(ForwardingTable::default());
12588 let supervisor_handle = SupervisorHandle::new();
12589 let supervisor =
12590 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
12591 .with_forwarding(Arc::clone(&forwarding))
12592 .with_handle(supervisor_handle.clone());
12593 let module = supervisor
12594 .supervise_configured(
12595 crate::ModuleSpec {
12596 module_id: "late-health-response".to_string(),
12597 program: PathBuf::from("disabled-module"),
12598 args: Vec::new(),
12599 env: Vec::new(),
12600 reserved: false,
12601 reserved_prefixes: Vec::new(),
12602 protocol: ModuleProtocol::Subc,
12603 overlap: Default::default(),
12604 },
12605 false,
12606 )
12607 .unwrap();
12608 let handler =
12609 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12610 .with_supervisor(supervisor_handle);
12611 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
12612 handler
12613 .handle_control_frame(
12614 &module_ctx,
12615 hello_frame_with_control_ops(
12616 "late-health-response",
12617 PROTOCOL_VERSION,
12618 7,
12619 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12620 ),
12621 )
12622 .await
12623 .unwrap();
12624 let probe_started_at = Instant::now() - Duration::from_millis(80);
12625 let pending = forwarding
12626 .begin_health_probe_rpc_for(
12627 "late-health-response",
12628 MODULE_CONTROL_OP_HEALTH_CHECK,
12629 probe_started_at,
12630 Instant::now() - Duration::from_millis(1),
12631 )
12632 .unwrap();
12633 assert!(forwarding
12634 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
12635 .unwrap());
12636
12637 let responses = handler
12638 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
12639 .await
12640 .unwrap();
12641
12642 assert!(responses.is_empty());
12643 let health = module.status().unwrap().health;
12644 assert_eq!(health.late_answer_count, 1);
12645 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
12646 }
12647
12648 #[tokio::test]
12649 async fn health_probe_timeout_and_module_death_are_typed() {
12650 let registry = Arc::new(Registry::default());
12651 let forwarding = Arc::new(ForwardingTable::default());
12652 let handler =
12653 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12654 .with_health_probe_timeout(Duration::from_millis(50));
12655 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12656 hello_via_sink(
12657 &handler,
12658 &module_ctx,
12659 &mut module_rx,
12660 hello_frame_with_control_ops(
12661 "aft",
12662 PROTOCOL_VERSION,
12663 7,
12664 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12665 ),
12666 )
12667 .await;
12668
12669 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12670 let responses = handler
12671 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
12672 .await
12673 .unwrap();
12674 assert_eq!(responses[0].header.ty, FrameType::Error);
12675 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
12676 let _ = module_rx.try_recv();
12677
12678 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
12679 let health_handler = handler.clone();
12680 let death_task = tokio::spawn(async move {
12681 health_handler
12682 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
12683 .await
12684 .unwrap()
12685 });
12686 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12687 .await
12688 .unwrap()
12689 .unwrap();
12690 handler
12691 .cleanup_connection(module_ctx.connection_id)
12692 .unwrap();
12693 let responses = death_task.await.unwrap();
12694 assert_eq!(responses[0].header.ty, FrameType::Error);
12695 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
12696 }
12697
12698 #[test]
12699 fn hello_requires_exact_protocol_version() {
12700 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
12701 let registry = Arc::new(Registry::default());
12702 let handler = ControlHandler::new(Arc::clone(®istry));
12703 let responses = handler
12704 .handle_control(
12705 ConnectionId::new(connection),
12706 hello_frame("aft", offered, 9),
12707 )
12708 .unwrap();
12709
12710 assert_eq!(responses.len(), 1);
12711 assert_eq!(responses[0].header.ty, FrameType::Error);
12712 let error = parse_error(&responses[0]);
12713 assert_eq!(error["code"], "version_unsupported");
12714 assert!(registry.get_module("aft").unwrap().is_none());
12715 assert_eq!(registry.active_registration_count().unwrap(), 0);
12716 }
12717 }
12718
12719 #[test]
12720 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
12721 let registry = Arc::new(Registry::default());
12722 let forwarding = Arc::new(ForwardingTable::default());
12723 let handler =
12724 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12725 let module_connection = ConnectionId::new(301);
12726 let registration = registry
12727 .register_with_control_ops(
12728 manifest("aft-push", PROTOCOL_VERSION),
12729 PROTOCOL_VERSION,
12730 module_connection,
12731 module_baseline_control_ops(),
12732 )
12733 .unwrap();
12734 let (module_tx, _module_rx) = mpsc::channel(8);
12735 let endpoint = forwarding
12736 .register_module_connection(
12737 module_connection,
12738 "aft-push".to_string(),
12739 PROTOCOL_VERSION,
12740 manifest_concurrency(®istration.manifest),
12741 FrameSink::new(module_tx),
12742 )
12743 .unwrap();
12744
12745 let unknown = Frame::build(
12747 FrameType::Push,
12748 control_flags(),
12749 0,
12750 0,
12751 5,
12752 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
12753 )
12754 .unwrap();
12755 let out = handler.handle_status_update(endpoint, unknown).unwrap();
12756 assert!(
12757 out.is_empty(),
12758 "unknown push op must be ignored, got {out:?}"
12759 );
12760
12761 let malformed = Frame::build(
12763 FrameType::Push,
12764 control_flags(),
12765 0,
12766 0,
12767 6,
12768 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
12769 )
12770 .unwrap();
12771 let out = handler.handle_status_update(endpoint, malformed).unwrap();
12772 assert_eq!(out.len(), 1);
12773 assert_eq!(out[0].header.ty, FrameType::Error);
12774 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
12775 }
12776
12777 #[test]
12778 fn hello_rejected_when_connection_already_owns_client_routes() {
12779 let registry = Arc::new(Registry::default());
12780 let forwarding = Arc::new(ForwardingTable::default());
12781 let handler =
12782 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12783 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
12785 let client_connection = ConnectionId::new(202);
12786
12787 let responses = handler
12790 .handle_control(
12791 client_connection,
12792 hello_frame("aft-second", PROTOCOL_VERSION, 9),
12793 )
12794 .unwrap();
12795 assert_eq!(responses[0].header.ty, FrameType::Error);
12796 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
12797 assert!(registry.get_module("aft-second").unwrap().is_none());
12798 }
12799
12800 #[tokio::test]
12801 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
12802 let registry = Arc::new(Registry::default());
12803 let forwarding = Arc::new(ForwardingTable::default());
12804 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
12805 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
12806 hello_via_sink(
12807 &handler,
12808 &module_ctx,
12809 &mut module_rx,
12810 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
12811 )
12812 .await;
12813 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
12814 let pending = forwarding
12815 .begin_route_bind_relay_for_test(
12816 client_ctx.connection_id,
12817 client_ctx.egress.clone(),
12818 2,
12819 "alpha",
12820 )
12821 .unwrap();
12822 forwarding
12823 .complete_pending_relay(
12824 module_ctx.connection_id,
12825 pending.corr,
12826 RouteBindRelayOutcome::Accepted,
12827 )
12828 .unwrap();
12829 client_rx.try_recv().unwrap();
12830 for module_id in ["beta", "alpha"] {
12831 let replies = handler
12832 .handle_control_frame(
12833 &module_ctx,
12834 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
12835 )
12836 .await
12837 .unwrap();
12838 assert_eq!(replies.len(), 1, "second HELLO must be refused");
12839 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
12840 }
12841 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
12842 assert!(registry.get_module("beta").unwrap().is_none());
12843 assert!(matches!(
12844 forwarding
12845 .lookup_data_route(
12846 client_ctx.connection_id,
12847 pending.client_channel,
12848 pending.client_epoch,
12849 )
12850 .unwrap(),
12851 DataRoute::Client(DataRouteState::Bound(_))
12852 ));
12853 assert!(handler
12854 .hello_launch_nonces
12855 .lock()
12856 .unwrap()
12857 .presented(module_ctx.connection_id, Some("alpha-nonce")));
12858 assert!(module_rx.try_recv().is_err());
12859 }
12860
12861 #[test]
12862 fn reserved_module_hello_requires_matching_launch_nonce() {
12863 let registry = Arc::new(Registry::default());
12864 let supervisor = SupervisorHandle::new();
12865 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12868 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12869
12870 let no_nonce = handler
12872 .handle_control(
12873 ConnectionId::new(1),
12874 hello_frame("vault", PROTOCOL_VERSION, 1),
12875 )
12876 .unwrap();
12877 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
12878 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
12879 assert!(registry.get_module("vault").unwrap().is_none());
12880
12881 let wrong = handler
12883 .handle_control(
12884 ConnectionId::new(2),
12885 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
12886 )
12887 .unwrap();
12888 assert_eq!(wrong[0].header.ty, FrameType::Error);
12889 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
12890 assert!(registry.get_module("vault").unwrap().is_none());
12891
12892 let ok = handler
12894 .handle_control(
12895 ConnectionId::new(3),
12896 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
12897 )
12898 .unwrap();
12899 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
12900 assert!(registry.get_module("vault").unwrap().is_some());
12901 }
12902
12903 #[test]
12904 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
12905 let registry = Arc::new(Registry::default());
12906 let supervisor = SupervisorHandle::new();
12907 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12908 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12909 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12910
12911 let squat = handler
12912 .handle_control(
12913 ConnectionId::new(1),
12914 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
12915 )
12916 .unwrap();
12917 assert_eq!(squat[0].header.ty, FrameType::Error);
12918 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
12919 assert!(parse_error(&squat[0])["message"]
12920 .as_str()
12921 .unwrap()
12922 .contains("fed:"));
12923
12924 let accepted_peer = handler
12925 .handle_control(
12926 ConnectionId::new(2),
12927 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
12928 )
12929 .unwrap();
12930 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
12931
12932 let accepted_short = handler
12933 .handle_control(
12934 ConnectionId::new(3),
12935 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
12936 )
12937 .unwrap();
12938 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
12939
12940 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
12941 let response = handler
12942 .handle_control(
12943 ConnectionId::new(conn),
12944 hello_frame(module_id, PROTOCOL_VERSION, conn),
12945 )
12946 .unwrap();
12947 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
12948 }
12949 }
12950
12951 #[test]
12952 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
12953 let registry = Arc::new(Registry::default());
12954 let supervisor = SupervisorHandle::new();
12955 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12956 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12957 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
12958 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12959
12960 let owner_nonce = handler
12961 .handle_control(
12962 ConnectionId::new(1),
12963 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
12964 )
12965 .unwrap();
12966 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
12967 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
12968 assert!(registry.get_module("fed:special").unwrap().is_none());
12969
12970 let exact_nonce = handler
12971 .handle_control(
12972 ConnectionId::new(2),
12973 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
12974 )
12975 .unwrap();
12976 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
12977 assert!(registry.get_module("fed:special").unwrap().is_some());
12978 }
12979
12980 #[test]
12981 fn non_reserved_module_ignores_launch_nonce() {
12982 let registry = Arc::new(Registry::default());
12983 let handler = ControlHandler::new(Arc::clone(®istry));
12986 let no_nonce = handler
12987 .handle_control(
12988 ConnectionId::new(1),
12989 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
12990 )
12991 .unwrap();
12992 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
12993 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
12994
12995 let echoed_nonce = handler
12996 .handle_control(
12997 ConnectionId::new(2),
12998 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
12999 )
13000 .unwrap();
13001 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
13002 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
13003 }
13004
13005 #[test]
13006 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
13007 let handler = ControlHandler::default();
13008 let conn = ConnectionId::new(1);
13009 let malformed = Frame::build(
13010 FrameType::Hello,
13011 control_flags(),
13012 0,
13013 0,
13014 3,
13015 b"{not json".to_vec(),
13016 )
13017 .unwrap();
13018
13019 let error = handler.handle_control(conn, malformed).unwrap();
13020 assert_eq!(error[0].header.ty, FrameType::Error);
13021 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
13022
13023 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13024 let pong = handler.handle_control(conn, ping).unwrap();
13025 assert_eq!(pong[0].header.ty, FrameType::Pong);
13026 assert_eq!(pong[0].header.corr, 4);
13027 }
13028
13029 #[test]
13030 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
13031 let registry = Arc::new(Registry::default());
13032 let handler = ControlHandler::new(Arc::clone(®istry));
13033
13034 handler
13035 .handle_control(
13036 ConnectionId::new(1),
13037 hello_frame("aft", PROTOCOL_VERSION, 1),
13038 )
13039 .unwrap();
13040 let duplicate = handler
13041 .handle_control(
13042 ConnectionId::new(2),
13043 hello_frame("aft", PROTOCOL_VERSION, 2),
13044 )
13045 .unwrap();
13046
13047 assert_eq!(duplicate[0].header.ty, FrameType::Error);
13048 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
13049 let registration = registry.get_module("aft").unwrap().unwrap();
13050 assert_eq!(registration.connection_id, ConnectionId::new(1));
13051 }
13052
13053 #[test]
13054 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
13055 let registry = Arc::new(Registry::default());
13056 let forwarding = Arc::new(ForwardingTable::default());
13057 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
13058 let handler =
13059 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13060 .with_process_liveness(process_liveness);
13061 let (ctx, route_channel, route_epoch) =
13062 bind_liveness_route(®istry, &forwarding, "aft-dead");
13063 let responses = handler
13064 .handle_route_poll(
13065 &ctx,
13066 route_poll_frame(41, PollKind::Liveness, route_channel),
13067 route_channel,
13068 route_epoch,
13069 PollKind::Liveness,
13070 )
13071 .unwrap();
13072
13073 assert_eq!(responses.len(), 1);
13074 assert_eq!(responses[0].header.ty, FrameType::Response);
13075 assert_route_poll_liveness(&responses[0], false);
13076 }
13077
13078 #[test]
13079 fn liveness_poll_without_process_source_uses_bound_route() {
13080 let registry = Arc::new(Registry::default());
13081 let forwarding = Arc::new(ForwardingTable::default());
13082 let handler =
13083 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13084 let (ctx, route_channel, route_epoch) =
13085 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
13086 let responses = handler
13087 .handle_route_poll(
13088 &ctx,
13089 route_poll_frame(42, PollKind::Liveness, route_channel),
13090 route_channel,
13091 route_epoch,
13092 PollKind::Liveness,
13093 )
13094 .unwrap();
13095
13096 assert_route_poll_liveness(&responses[0], true);
13097 }
13098
13099 #[test]
13100 fn liveness_poll_untracked_process_source_uses_bound_route() {
13101 let registry = Arc::new(Registry::default());
13102 let forwarding = Arc::new(ForwardingTable::default());
13103 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
13104 let handler =
13105 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13106 .with_process_liveness(process_liveness);
13107 let (ctx, route_channel, route_epoch) =
13108 bind_liveness_route(®istry, &forwarding, "aft-untracked");
13109 let responses = handler
13110 .handle_route_poll(
13111 &ctx,
13112 route_poll_frame(43, PollKind::Liveness, route_channel),
13113 route_channel,
13114 route_epoch,
13115 PollKind::Liveness,
13116 )
13117 .unwrap();
13118
13119 assert_route_poll_liveness(&responses[0], true);
13120 }
13121
13122 #[tokio::test]
13123 async fn unknown_op_returns_unknown_control_op() {
13124 let handler = ControlHandler::default();
13125 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
13126 let request = Frame::build(
13127 FrameType::Request,
13128 control_flags(),
13129 0,
13130 0,
13131 55,
13132 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
13133 )
13134 .unwrap();
13135
13136 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13137
13138 assert_eq!(response.len(), 1);
13139 assert_eq!(response[0].header.ty, FrameType::Error);
13140 assert_eq!(response[0].header.corr, 55);
13141 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
13142 }
13143
13144 #[tokio::test]
13145 async fn supervisor_provenance_rejects_unknown_exact_module() {
13146 let handler = ControlHandler::default();
13147 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
13148 let request = Frame::build(
13149 FrameType::Request,
13150 control_flags(),
13151 0,
13152 0,
13153 57,
13154 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
13155 )
13156 .unwrap();
13157
13158 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13159
13160 assert_eq!(response.len(), 1);
13161 assert_eq!(response[0].header.ty, FrameType::Error);
13162 assert_eq!(response[0].header.corr, 57);
13163 let error = parse_error(&response[0]);
13164 assert_eq!(error["code"], "unknown_module");
13165 assert_eq!(error["message"], "module_id 'missing' is not supervised");
13166 }
13167
13168 #[test]
13169 fn provenance_probe_override_keeps_handler_tests_deterministic() {
13170 let expected = subc_control::RunningImageAgreement::Unavailable {
13171 reason: subc_control::RunningImageUnavailableReason::HashFailed,
13172 };
13173 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
13174 assert_eq!(handler.provenance_probe_override, Some(expected));
13175 }
13176
13177 #[test]
13178 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
13179 let verdict = reload_verdict(
13180 std::path::Path::new("/bin/new"),
13181 Some(std::path::Path::new("/bin/old")),
13182 subc_control::RunningImageAgreement::Unavailable {
13183 reason: subc_control::RunningImageUnavailableReason::HashFailed,
13184 },
13185 );
13186 assert!(matches!(
13187 verdict.path,
13188 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
13189 if configured == std::path::Path::new("/bin/new")
13190 && spawned_from == std::path::Path::new("/bin/old")
13191 ));
13192 }
13193
13194 #[test]
13195 fn launch_spec_without_running_digest_is_unavailable_not_match() {
13196 use subc_control::{
13197 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
13198 };
13199 let configured = crate::supervise::ModuleSpec {
13200 module_id: "legacy".into(),
13201 program: "/module".into(),
13202 args: Vec::new(),
13203 env: Vec::new(),
13204 reserved: false,
13205 reserved_prefixes: Vec::new(),
13206 protocol: subc_control::ModuleProtocol::Subc,
13207 overlap: Default::default(),
13208 };
13209 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
13210 assert_eq!(
13211 launch_spec_agreement(&configured, false, Some(&digest)),
13212 Agreement::Unavailable {
13213 reason: Reason::NotRunning
13214 }
13215 );
13216 assert_eq!(
13217 launch_spec_agreement(&configured, true, None),
13218 Agreement::Unavailable {
13219 reason: Reason::SpawnedSpecUnavailable
13220 }
13221 );
13222 assert_eq!(
13223 launch_spec_agreement(&configured, true, Some(&digest)),
13224 Agreement::Match
13225 );
13226 }
13227
13228 #[cfg(windows)]
13229 #[test]
13230 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
13231 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
13232 let configured = dir.join("module.exe");
13233 std::fs::write(&configured, b"executable path fixture").unwrap();
13234 let spawned = std::fs::canonicalize(&configured).unwrap();
13235 assert_ne!(
13236 configured, spawned,
13237 "fixture must exercise different path spellings"
13238 );
13239 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
13240 let lower_case = configured.to_string_lossy().to_lowercase();
13241 for spelling in [
13242 &configured,
13243 Path::new(&forward_slashes),
13244 Path::new(&lower_case),
13245 ] {
13246 let verdict = reload_verdict(
13247 spelling,
13248 Some(&spawned),
13249 subc_control::RunningImageAgreement::Unavailable {
13250 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13251 },
13252 );
13253 assert_eq!(
13254 verdict.path,
13255 ReloadPathAgreement::Match,
13256 "configured {spelling:?}, spawned {spawned:?}"
13257 );
13258 }
13259 }
13260
13261 #[cfg(windows)]
13262 #[test]
13263 fn windows_reload_path_keeps_different_existing_program_pending() {
13264 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
13265 let old = dir.join("old.exe");
13266 let new = dir.join("new.exe");
13267 std::fs::write(&old, b"same executable content").unwrap();
13268 std::fs::copy(&old, &new).unwrap();
13269 let spawned = std::fs::canonicalize(&old).unwrap();
13270 let image = subc_control::RunningImageAgreement::Unavailable {
13271 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13272 };
13273 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
13274 assert!(
13275 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
13276 if configured == new && spawned_from == spawned)
13277 );
13278 let missing = dir.join("missing.exe");
13279 assert!(matches!(
13280 reload_verdict(&missing, Some(&spawned), image).path,
13281 ReloadPathAgreement::Mismatch { .. }
13282 ));
13283 }
13284
13285 #[test]
13286 fn reload_verdict_detects_replaced_image_at_same_path() {
13287 let image = subc_control::RunningImageAgreement::Mismatch {
13288 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
13289 digest: "old".into(),
13290 },
13291 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
13292 digest: "new".into(),
13293 },
13294 };
13295 let verdict = reload_verdict(
13296 std::path::Path::new("/bin/same"),
13297 Some(std::path::Path::new("/bin/same")),
13298 image.clone(),
13299 );
13300 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13301 assert_eq!(verdict.image, image);
13302 }
13303
13304 #[test]
13305 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
13306 let image = subc_control::RunningImageAgreement::Unavailable {
13307 reason: subc_control::RunningImageUnavailableReason::NotRunning,
13308 };
13309 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
13310 assert_eq!(
13311 verdict.path,
13312 subc_control::ReloadPathAgreement::Unavailable {
13313 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
13314 }
13315 );
13316 assert_eq!(verdict.image, image);
13317
13318 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
13319 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13320 };
13321 let verdict = reload_verdict(
13322 std::path::Path::new("/bin/same"),
13323 Some(std::path::Path::new("/bin/same")),
13324 unconfirmed.clone(),
13325 );
13326 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13327 assert_eq!(verdict.image, unconfirmed);
13328 }
13329
13330 #[test]
13331 fn reload_verdict_preserves_each_image_unavailability_reason() {
13332 use subc_control::RunningImageUnavailableReason as Reason;
13333
13334 for reason in [
13335 Reason::NotRunning,
13336 Reason::UnsupportedPlatform,
13337 Reason::RunningExecutableUnreadable,
13338 Reason::SpawnedPathUnreadable,
13339 Reason::HashFailed,
13340 Reason::ProcessIdentityUnconfirmed,
13341 Reason::Unknown("future_probe_reason".to_string()),
13342 ] {
13343 let image = subc_control::RunningImageAgreement::Unavailable {
13344 reason: reason.clone(),
13345 };
13346 let verdict = reload_verdict(
13347 std::path::Path::new("/bin/same"),
13348 Some(std::path::Path::new("/bin/same")),
13349 image.clone(),
13350 );
13351 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13352 assert_eq!(verdict.image, image, "{reason:?}");
13353 }
13354 }
13355
13356 #[tokio::test]
13357 async fn malformed_control_bodies_return_invalid_control_body() {
13358 let handler = ControlHandler::default();
13359 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
13360
13361 for (corr, body) in [
13362 (56, br#"{"route_channel":1}"#.as_slice()),
13363 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
13364 (
13365 58,
13366 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
13367 ),
13368 ] {
13369 let request = Frame::build(
13370 FrameType::Request,
13371 control_flags(),
13372 0,
13373 0,
13374 corr,
13375 body.to_vec(),
13376 )
13377 .unwrap();
13378 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13379
13380 assert_eq!(response.len(), 1);
13381 assert_eq!(response[0].header.ty, FrameType::Error);
13382 assert_eq!(response[0].header.corr, corr);
13383 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
13384 }
13385 }
13386
13387 #[tokio::test]
13388 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
13389 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13390 let registry = Arc::new(Registry::default());
13391 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13392 let router = Router::with_control_handler(Arc::clone(&control));
13393 let connection = router.begin_connection();
13394 let (ctx, mut rx) = route_ctx(connection.id());
13395
13396 router
13397 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
13398 .await
13399 .unwrap();
13400 let response = rx.recv().await.unwrap();
13401 let ack = parse_ack(&response);
13402 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13403 let channel = 1;
13404
13405 let goodbye =
13406 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
13407 router.route_for_connection(&ctx, goodbye).await.unwrap();
13408 assert!(rx.try_recv().is_err());
13409 assert!(registry.get_module("aft").unwrap().is_none());
13410
13411 router
13412 .route_for_connection(&ctx, channel_request(channel, 13))
13413 .await
13414 .unwrap();
13415 let error_frame = rx.recv().await.unwrap();
13416 assert_eq!(error_frame.header.ty, FrameType::Error);
13417 assert_eq!(error_frame.header.channel, channel);
13418 let captured = crate::router::test_log::captured_logs(&logs);
13419 assert_eq!(
13420 captured
13421 .lines()
13422 .filter(|line| {
13423 line.contains("module_id=aft")
13424 && line.contains("reason=explicit_goodbye")
13425 && line.contains("module registration ended")
13426 })
13427 .count(),
13428 1,
13429 "unexpected GOODBYE registry log: {captured}"
13430 );
13431 }
13432
13433 #[tokio::test]
13434 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
13435 let registry = Arc::new(Registry::default());
13436 let handler = ControlHandler::new(registry).with_capability_config(
13437 [("prov".to_string(), true), ("cons".to_string(), true)],
13438 BTreeMap::new(),
13439 );
13440 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
13441 register_capability_manifest(
13442 &handler,
13443 &provider_ctx,
13444 &mut provider_rx,
13445 capability_manifest("prov", &["thing/v1"], &[]),
13446 1,
13447 )
13448 .await;
13449 let mut consumer = capability_manifest("cons", &[], &[]);
13450 consumer.capabilities.as_mut().unwrap().requires.push(
13451 subc_protocol::manifest::CapabilityRequirement {
13452 capability: "thing/v1".to_string(),
13453 need: subc_protocol::manifest::CapabilityNeed::Required,
13454 },
13455 );
13456 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
13457 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
13458 assert_eq!(
13459 handler.capability_evaluator.verdict("cons", "thing/v1"),
13460 Some(CapabilityVerdict::Provided)
13461 );
13462 let (mut client_rx, _) = open_route_for_capability_test(
13463 &handler,
13464 &provider_ctx,
13465 &mut provider_rx,
13466 703,
13467 3,
13468 "prov",
13469 None,
13470 )
13471 .await;
13472 let goodbye =
13473 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13474 handler
13475 .handle_control_frame(&provider_ctx, goodbye)
13476 .await
13477 .unwrap();
13478 assert_eq!(
13479 handler.capability_evaluator.verdict("cons", "thing/v1"),
13480 Some(CapabilityVerdict::NeverProvided)
13481 );
13482 let closed = client_rx
13483 .try_recv()
13484 .expect("GOODBYE pushes route.closed before route GOODBYE");
13485 assert!(
13486 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
13487 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
13488 );
13489 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
13490 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
13491 }
13492
13493 struct RouteClosedFixture {
13494 handler: ControlHandler,
13495 #[cfg(unix)]
13496 supervisor: Supervisor,
13497 module_ctx: RouteCtx,
13498 #[cfg(unix)]
13499 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13500 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13501 route: Arc<crate::forwarding::RouteBinding>,
13502 }
13503
13504 async fn route_closed_fixture() -> RouteClosedFixture {
13505 let registry = Arc::new(Registry::default());
13506 let handler = ControlHandler::new(Arc::clone(®istry));
13507 #[cfg(unix)]
13508 let supervisor = Supervisor::new(registry, RestartPolicy::default())
13509 .with_forwarding(Arc::clone(&handler.forwarding));
13510 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
13511 register_capability_manifest(
13512 &handler,
13513 &module_ctx,
13514 &mut module_rx,
13515 capability_manifest("prov", &[], &[]),
13516 1,
13517 )
13518 .await;
13519 let (client_rx, bind) = open_route_for_capability_test(
13520 &handler,
13521 &module_ctx,
13522 &mut module_rx,
13523 702,
13524 2,
13525 "prov",
13526 None,
13527 )
13528 .await;
13529 let ModuleControlRequest::RouteBind {
13530 route_channel: channel,
13531 epoch: route_epoch,
13532 ..
13533 } = bind
13534 else {
13535 panic!("expected route.bind");
13536 };
13537 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
13538 handler
13539 .forwarding
13540 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
13541 .unwrap()
13542 else {
13543 panic!("the acknowledged route must be bound");
13544 };
13545 RouteClosedFixture {
13546 handler,
13547 #[cfg(unix)]
13548 supervisor,
13549 module_ctx,
13550 #[cfg(unix)]
13551 module_rx,
13552 client_rx,
13553 route,
13554 }
13555 }
13556
13557 impl RouteClosedFixture {
13558 #[cfg(unix)]
13559 async fn cut(&mut self) {
13560 let notices = async {
13561 let mut draining = self.module_rx.recv().await.unwrap();
13562 assert!(matches!(
13563 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
13564 &draining.body
13565 )
13566 .unwrap(),
13567 subc_protocol::session::ModuleControlCommand::Draining {
13568 reason: RouteCloseReason::Restart,
13569 ..
13570 }
13571 ));
13572 draining.flushed.take().unwrap().send(()).unwrap();
13573 let mut closing = self.client_rx.recv().await.unwrap();
13574 assert!(matches!(
13575 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
13576 ClientControlPush::RouteClosing {
13577 reason: RouteCloseReason::Restart,
13578 ..
13579 }
13580 ));
13581 closing.flushed.take().unwrap().send(()).unwrap();
13582 };
13583 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
13584 result.unwrap();
13585 }
13586
13587 fn disconnect(&mut self) -> ClientControlPush {
13588 self.handler
13589 .cleanup_connection(self.module_ctx.connection_id)
13590 .unwrap();
13591 let closed = self.client_rx.try_recv().unwrap();
13592 assert_eq!(closed.header.ty, FrameType::Push);
13593 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
13594 assert!(matches!(
13595 &push,
13596 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
13597 if channels == &[self.route.client_channel]
13598 ));
13599 assert_eq!(
13600 self.client_rx.try_recv().unwrap().header.ty,
13601 FrameType::Goodbye
13602 );
13603 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
13604 push
13605 }
13606 }
13607
13608 #[cfg(unix)]
13609 #[tokio::test(start_paused = true)]
13610 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
13611 let mut fixture = route_closed_fixture().await;
13612 fixture.cut().await;
13613 assert!(matches!(
13614 fixture.disconnect(),
13615 ClientControlPush::RouteClosed {
13616 reason: RouteCloseReason::Restart,
13617 drained: true,
13618 ..
13619 }
13620 ));
13621 }
13622
13623 #[cfg(unix)]
13624 #[tokio::test(start_paused = true)]
13625 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
13626 let mut fixture = route_closed_fixture().await;
13627 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
13628 let started = tokio::time::Instant::now();
13629 fixture.cut().await;
13630 assert!(started.elapsed() >= Duration::from_secs(2));
13631 assert_eq!(fixture.route.flow.in_flight(), 1);
13632 assert!(matches!(
13633 fixture.disconnect(),
13634 ClientControlPush::RouteClosed {
13635 reason: RouteCloseReason::Restart,
13636 drained: false,
13637 ..
13638 }
13639 ));
13640 }
13641
13642 #[tokio::test]
13643 async fn module_connection_crash_reports_crash_and_not_drained() {
13644 let mut fixture = route_closed_fixture().await;
13645 assert!(matches!(
13649 fixture.disconnect(),
13650 ClientControlPush::RouteClosed {
13651 reason: RouteCloseReason::Crash,
13652 drained: false,
13653 ..
13654 }
13655 ));
13656 }
13657
13658 #[tokio::test]
13659 async fn dropping_router_connection_releases_registration() {
13660 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13661 let registry = Arc::new(Registry::default());
13662 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13663 let router = Router::with_control_handler(Arc::clone(&control));
13664 let connection = router.begin_connection();
13665 let connection_id = connection.id();
13666 let (ctx, mut rx) = route_ctx(connection_id);
13667
13668 router
13669 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
13670 .await
13671 .unwrap();
13672 let response = rx.recv().await.unwrap();
13673 let ack = parse_ack(&response);
13674 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13675 assert!(registry.get_module("aft").unwrap().is_some());
13676
13677 drop(connection);
13678
13679 assert!(registry.get_module("aft").unwrap().is_none());
13680 assert_eq!(registry.active_registration_count().unwrap(), 0);
13681
13682 control
13683 .cleanup_connection(ConnectionId::new(u64::MAX))
13684 .unwrap();
13685 let captured = crate::router::test_log::captured_logs(&logs);
13686 println!("captured registry lifecycle logs:\n{captured}");
13687 let events: Vec<_> = captured
13688 .lines()
13689 .filter(|line| {
13690 line.contains("module registered module_id=aft ")
13691 || line.contains("module registration ended")
13692 })
13693 .collect();
13694 assert_eq!(
13695 events.len(),
13696 2,
13697 "unexpected registry lifecycle logs: {captured}"
13698 );
13699 assert!(events[0].contains("module registered module_id=aft "));
13700 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
13701 assert!(events[1].contains(&format!(
13702 "module_id=aft connection_id={}",
13703 connection_id.get()
13704 )));
13705 assert!(events[1].contains("reason=connection_closed"));
13706 assert!(events[1].contains("module registration ended"));
13707 assert_eq!(
13708 captured
13709 .lines()
13710 .filter(|line| line.contains("module registered module_id=aft "))
13711 .count(),
13712 1,
13713 "legacy registration admission line must appear once: {captured}"
13714 );
13715 }
13716
13717 #[tokio::test]
13718 async fn hello_registration_keeps_legacy_module_registered_line_once() {
13719 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13720 let registry = Arc::new(Registry::default());
13721 let control = ControlHandler::new(Arc::clone(®istry));
13722 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
13723 hello_via_sink(
13724 &control,
13725 &ctx,
13726 &mut rx,
13727 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
13728 )
13729 .await;
13730
13731 let captured = crate::router::test_log::captured_logs(&logs);
13732 let admissions: Vec<_> = captured
13733 .lines()
13734 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
13735 .collect();
13736 assert_eq!(
13737 admissions.len(),
13738 1,
13739 "legacy admission line must remain exactly once: {captured}"
13740 );
13741 assert!(admissions[0].contains("routable_provider=true"));
13742 assert!(admissions[0].contains("connection_id=777"));
13743 }
13744
13745 fn capability_manifest(
13746 module_id: &str,
13747 provides: &[&str],
13748 must_never_reach: &[&str],
13749 ) -> ModuleManifest {
13750 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
13751 manifest.capabilities = Some(CapabilityDeclarations {
13752 provides: provides
13753 .iter()
13754 .map(|capability| (*capability).to_string())
13755 .collect(),
13756 requires: Vec::new(),
13757 must_never_reach: must_never_reach
13758 .iter()
13759 .map(|capability| (*capability).to_string())
13760 .collect(),
13761 });
13762 manifest
13763 }
13764
13765 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
13766 Frame::build(
13767 FrameType::Hello,
13768 control_flags(),
13769 0,
13770 0,
13771 corr,
13772 serde_json::to_vec(&ModuleHelloBody {
13773 protocol_ver: manifest.protocol_ver,
13774 manifest,
13775 control_ops: None,
13776 launch_nonce: None,
13777 })
13778 .expect("capability test HELLO serializes"),
13779 )
13780 .expect("capability test HELLO frame builds")
13781 }
13782
13783 fn catalog_update_with_capabilities_frame(
13784 corr: u64,
13785 capabilities: CapabilityDeclarations,
13786 ) -> Frame {
13787 Frame::build(
13788 FrameType::Request,
13789 control_flags(),
13790 0,
13791 0,
13792 corr,
13793 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13794 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
13795 capabilities: Some(capabilities),
13796 ready: None,
13797 })
13798 .expect("capability catalog.update serializes"),
13799 )
13800 .expect("capability catalog.update frame builds")
13801 }
13802
13803 async fn register_capability_manifest(
13804 handler: &ControlHandler,
13805 ctx: &RouteCtx,
13806 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13807 manifest: ModuleManifest,
13808 corr: u64,
13809 ) {
13810 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
13811 }
13812
13813 async fn open_route_for_capability_test(
13814 handler: &ControlHandler,
13815 target_ctx: &RouteCtx,
13816 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13817 client_connection_id: u64,
13818 corr: u64,
13819 target_module_id: &str,
13820 consumer_identity: Option<ConsumerIdentity>,
13821 ) -> (
13822 mpsc::Receiver<crate::router::OutboundFrame>,
13823 ModuleControlRequest,
13824 ) {
13825 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
13826 let route_handler = handler.clone();
13827 let target_module_id = target_module_id.to_string();
13828 let route_task = tokio::spawn(async move {
13829 route_handler
13830 .handle_control_frame(
13831 &client_ctx,
13832 route_open_frame_with_admission_facts(
13833 corr,
13834 &target_module_id,
13835 unique_project_root("admission-facts"),
13836 consumer_identity,
13837 None,
13838 ),
13839 )
13840 .await
13841 .expect("capability test route.open succeeds")
13842 });
13843 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
13844 .await
13845 .expect("capability test route.open must reach route.bind")
13846 .expect("target control receiver stays open");
13847 let bind_request: ModuleControlRequest =
13848 serde_json::from_slice(&bind.body).expect("route.bind decodes");
13849 handler
13850 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
13851 .await
13852 .expect("capability test route.bind ACK succeeds");
13853 assert!(route_task.await.expect("route.open task joins").is_empty());
13854 let opened = client_rx
13855 .recv()
13856 .await
13857 .expect("successful route.open publishes a response");
13858 assert!(matches!(
13859 serde_json::from_slice::<ClientControlResponse>(&opened.body),
13860 Ok(ClientControlResponse::RouteOpen { .. })
13861 ));
13862 (client_rx, bind_request)
13863 }
13864
13865 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
13866 assert_eq!(frame.header.ty, FrameType::Push);
13867 assert_eq!(frame.header.channel, 0);
13868 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
13869 .expect("route.closed control push decodes");
13870 let ClientControlPush::RouteClosed { channels, .. } = &push else {
13871 panic!("expected route.closed");
13872 };
13873 assert_eq!(channels.len(), 1, "exactly one violating route closed");
13874 let channels = channels.clone();
13875 assert_eq!(
13876 push,
13877 ClientControlPush::RouteClosed {
13878 module_id: target_module_id.to_string(),
13879 channels,
13880 reason: RouteCloseReason::CapabilityDenied,
13881 drained: false,
13882 abandoned: 0,
13883 excluded_subscriptions: 0,
13884 terminal: Some(false),
13885 }
13886 );
13887 }
13888
13889 #[tokio::test]
13890 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
13891 let registry = Arc::new(Registry::default());
13892 let forwarding = Arc::new(ForwardingTable::default());
13893 let supervisor = SupervisorHandle::new();
13894 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13895 let handler =
13896 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13897 .with_supervisor(supervisor);
13898 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
13899 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
13900 register_capability_manifest(
13901 &handler,
13902 &target_ctx,
13903 &mut target_rx,
13904 capability_manifest("target", &["credentials-provider/v1"], &[]),
13905 1,
13906 )
13907 .await;
13908 register_capability_manifest(
13909 &handler,
13910 &opener_ctx,
13911 &mut opener_rx,
13912 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13913 2,
13914 )
13915 .await;
13916
13917 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
13918 let replies = handler
13919 .handle_control_frame(
13920 &client_ctx,
13921 route_open_frame_with_admission_facts(
13922 3,
13923 "target",
13924 unique_project_root("admission-facts"),
13925 Some(ConsumerIdentity {
13926 module_id: "opener".to_string(),
13927 launch_nonce: "opener-nonce".to_string(),
13928 }),
13929 None,
13930 ),
13931 )
13932 .await
13933 .expect("denied route.open returns a typed frame");
13934 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13935 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13936 assert!(
13937 target_rx.try_recv().is_err(),
13938 "forbidden route.open must not relay route.bind"
13939 );
13940 }
13941
13942 #[tokio::test]
13943 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
13944 let registry = Arc::new(Registry::default());
13945 let forwarding = Arc::new(ForwardingTable::default());
13946 let supervisor = SupervisorHandle::new();
13947 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13948 let handler =
13949 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13950 .with_supervisor(supervisor);
13951 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
13952 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
13953 register_capability_manifest(
13954 &handler,
13955 &target_ctx,
13956 &mut target_rx,
13957 capability_manifest("target", &["credentials-provider/v1"], &[]),
13958 1,
13959 )
13960 .await;
13961 register_capability_manifest(
13962 &handler,
13963 &old_opener_ctx,
13964 &mut old_opener_rx,
13965 capability_manifest("opener", &[], &[]),
13966 2,
13967 )
13968 .await;
13969 let (mut client_rx, _) = open_route_for_capability_test(
13970 &handler,
13971 &target_ctx,
13972 &mut target_rx,
13973 712,
13974 3,
13975 "target",
13976 Some(ConsumerIdentity {
13977 module_id: "opener".to_string(),
13978 launch_nonce: "opener-nonce".to_string(),
13979 }),
13980 )
13981 .await;
13982 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13983
13984 handler
13985 .cleanup_connection(old_opener_ctx.connection_id)
13986 .expect("old opener registration cleans up");
13987 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
13988 register_capability_manifest(
13989 &handler,
13990 &new_opener_ctx,
13991 &mut new_opener_rx,
13992 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13993 4,
13994 )
13995 .await;
13996
13997 assert_capability_denied_push(
13998 client_rx
13999 .try_recv()
14000 .expect("HELLO deny addition must emit route.closed")
14001 .frame,
14002 "target",
14003 );
14004 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14005 assert!(matches!(
14006 target_rx.try_recv(),
14007 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14008 ));
14009 }
14010
14011 #[tokio::test]
14012 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
14013 let registry = Arc::new(Registry::default());
14014 let forwarding = Arc::new(ForwardingTable::default());
14015 let supervisor = SupervisorHandle::new();
14016 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14017 let handler =
14018 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14019 .with_supervisor(supervisor);
14020 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
14021 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
14022 register_capability_manifest(
14023 &handler,
14024 &target_ctx,
14025 &mut target_rx,
14026 capability_manifest("target", &[], &[]),
14027 1,
14028 )
14029 .await;
14030 register_capability_manifest(
14031 &handler,
14032 &opener_ctx,
14033 &mut opener_rx,
14034 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14035 2,
14036 )
14037 .await;
14038 let (mut client_rx, _) = open_route_for_capability_test(
14039 &handler,
14040 &target_ctx,
14041 &mut target_rx,
14042 722,
14043 3,
14044 "target",
14045 Some(ConsumerIdentity {
14046 module_id: "opener".to_string(),
14047 launch_nonce: "opener-nonce".to_string(),
14048 }),
14049 )
14050 .await;
14051 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14052
14053 let replies = handler
14054 .handle_control_frame(
14055 &target_ctx,
14056 catalog_update_with_capabilities_frame(
14057 4,
14058 CapabilityDeclarations {
14059 provides: vec!["credentials-provider/v1".to_string()],
14060 requires: Vec::new(),
14061 must_never_reach: Vec::new(),
14062 },
14063 ),
14064 )
14065 .await
14066 .expect("claim catalog.update succeeds");
14067 assert!(matches!(
14068 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
14069 Ok(ModuleControlResponseToModule::CatalogUpdate {})
14070 ));
14071 assert_capability_denied_push(
14072 client_rx
14073 .try_recv()
14074 .expect("claim addition must emit route.closed")
14075 .frame,
14076 "target",
14077 );
14078 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14079 assert!(matches!(
14080 target_rx.try_recv(),
14081 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
14082 ));
14083 }
14084
14085 #[tokio::test]
14086 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
14087 let registry = Arc::new(Registry::default());
14088 let forwarding = Arc::new(ForwardingTable::default());
14089 let supervisor = SupervisorHandle::new();
14090 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14091 let handler =
14092 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14093 .with_supervisor(supervisor);
14094 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
14095 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
14096 register_capability_manifest(
14097 &handler,
14098 &target_ctx,
14099 &mut target_rx,
14100 capability_manifest("target", &["credentials-provider/v1"], &[]),
14101 1,
14102 )
14103 .await;
14104 register_capability_manifest(
14105 &handler,
14106 &opener_ctx,
14107 &mut opener_rx,
14108 capability_manifest("opener", &[], &[]),
14109 2,
14110 )
14111 .await;
14112 let (mut client_rx, _) = open_route_for_capability_test(
14113 &handler,
14114 &target_ctx,
14115 &mut target_rx,
14116 732,
14117 3,
14118 "target",
14119 Some(ConsumerIdentity {
14120 module_id: "opener".to_string(),
14121 launch_nonce: "opener-nonce".to_string(),
14122 }),
14123 )
14124 .await;
14125 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14126
14127 handler
14128 .handle_control_frame(
14129 &target_ctx,
14130 catalog_update_with_capabilities_frame(
14131 4,
14132 CapabilityDeclarations {
14133 provides: Vec::new(),
14134 requires: Vec::new(),
14135 must_never_reach: Vec::new(),
14136 },
14137 ),
14138 )
14139 .await
14140 .expect("claim removal catalog.update succeeds");
14141 assert_eq!(
14142 forwarding.active_binding_count().unwrap(),
14143 1,
14144 "removing an attested target claim must leave the route census unchanged"
14145 );
14146 assert!(
14147 client_rx.try_recv().is_err(),
14148 "claim removal must not emit route.closed capability_denied"
14149 );
14150 assert!(
14151 target_rx.try_recv().is_err(),
14152 "claim removal must not send the target a route GOODBYE"
14153 );
14154 }
14155
14156 #[tokio::test]
14159 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
14160 let registry = Arc::new(Registry::default());
14161 let forwarding = Arc::new(ForwardingTable::default());
14162 let supervisor = SupervisorHandle::new();
14163 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
14164 let handler =
14165 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14166 .with_supervisor(supervisor);
14167 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
14168 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
14169 register_capability_manifest(
14170 &handler,
14171 &target_ctx,
14172 &mut target_rx,
14173 capability_manifest("target", &["credentials-provider/v1"], &[]),
14174 1,
14175 )
14176 .await;
14177 register_capability_manifest(
14178 &handler,
14179 &opener_ctx,
14180 &mut opener_rx,
14181 capability_manifest("opener", &[], &["credentials-provider/v1"]),
14182 2,
14183 )
14184 .await;
14185
14186 let (_client_rx, bind) = open_route_for_capability_test(
14187 &handler,
14188 &target_ctx,
14189 &mut target_rx,
14190 742,
14191 3,
14192 "target",
14193 None,
14194 )
14195 .await;
14196 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
14197 panic!("direct scope-honesty route must bind");
14198 };
14199 assert_eq!(principal, Some(Principal::Direct));
14200 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
14201 }
14202
14203 #[tokio::test]
14206 async fn must_never_reach_self_route_is_capability_forbidden() {
14207 let registry = Arc::new(Registry::default());
14208 let forwarding = Arc::new(ForwardingTable::default());
14209 let supervisor = SupervisorHandle::new();
14210 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
14211 let handler =
14212 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14213 .with_supervisor(supervisor);
14214 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
14215 register_capability_manifest(
14216 &handler,
14217 &self_ctx,
14218 &mut self_rx,
14219 capability_manifest(
14220 "self-provider",
14221 &["credentials-provider/v1"],
14222 &["credentials-provider/v1"],
14223 ),
14224 1,
14225 )
14226 .await;
14227
14228 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
14229 let replies = handler
14230 .handle_control_frame(
14231 &client_ctx,
14232 route_open_frame_with_admission_facts(
14233 2,
14234 "self-provider",
14235 unique_project_root("admission-facts"),
14236 Some(ConsumerIdentity {
14237 module_id: "self-provider".to_string(),
14238 launch_nonce: "self-nonce".to_string(),
14239 }),
14240 None,
14241 ),
14242 )
14243 .await
14244 .expect("self-route refusal returns a typed frame");
14245 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14246 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14247 assert!(
14248 self_rx.try_recv().is_err(),
14249 "self denial must not relay route.bind"
14250 );
14251 }
14252
14253 #[test]
14254 fn unsupported_channel_zero_frame_returns_error() {
14255 let handler = ControlHandler::default();
14256 let request = Frame::build(
14257 FrameType::Request,
14258 control_flags(),
14259 0,
14260 0,
14261 21,
14262 b"opaque".to_vec(),
14263 )
14264 .unwrap();
14265
14266 let response = handler
14267 .handle_control(ConnectionId::new(1), request)
14268 .unwrap();
14269
14270 assert_eq!(response[0].header.ty, FrameType::Error);
14271 assert_eq!(
14272 parse_error(&response[0])["code"],
14273 "unsupported_control_frame"
14274 );
14275 }
14276
14277 mod swap {
14282 use super::*;
14283
14284 const INCUMBENT: ConnectionId = ConnectionId::new(30);
14285 const CANDIDATE: ConnectionId = ConnectionId::new(40);
14286
14287 struct Swap {
14288 registry: Arc<Registry>,
14289 forwarding: Arc<ForwardingTable>,
14290 handler: ControlHandler,
14291 incumbent_ctx: RouteCtx,
14292 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14293 candidate_ctx: RouteCtx,
14294 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14295 }
14296
14297 async fn swap_with_incumbent() -> Swap {
14298 let registry = Arc::new(Registry::default());
14299 let forwarding = Arc::new(ForwardingTable::default());
14300 let handler =
14301 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14302 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
14303 hello_via_sink(
14304 &handler,
14305 &incumbent_ctx,
14306 &mut incumbent_rx,
14307 hello_frame("aft", PROTOCOL_VERSION, 7),
14308 )
14309 .await;
14310 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
14311 Swap {
14312 registry,
14313 forwarding,
14314 handler,
14315 incumbent_ctx,
14316 incumbent_rx,
14317 candidate_ctx,
14318 candidate_rx,
14319 }
14320 }
14321
14322 fn register_candidate(swap: &Swap, ready: Option<bool>) {
14323 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
14324 candidate_manifest.ready = ready;
14325 let registration = swap
14326 .registry
14327 .register_candidate_with_control_ops(
14328 candidate_manifest,
14329 PROTOCOL_VERSION,
14330 CANDIDATE,
14331 module_baseline_control_ops(),
14332 )
14333 .unwrap();
14334 swap.forwarding
14335 .register_candidate_module_connection(
14336 CANDIDATE,
14337 "aft".to_string(),
14338 PROTOCOL_VERSION,
14339 manifest_concurrency(®istration.manifest),
14340 swap.candidate_ctx.egress.clone(),
14341 )
14342 .unwrap();
14343 }
14344
14345 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
14346 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
14347 swap.registry.promote_candidate("aft").unwrap().unwrap();
14348 cutover.incumbent.unwrap()
14349 }
14350
14351 #[tokio::test]
14360 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
14361 let mut swap = swap_with_incumbent().await;
14362 let hold = Arc::new(std::sync::Barrier::new(2));
14363 let handler = swap
14364 .handler
14365 .clone()
14366 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14367 handler
14368 .supervisor
14369 .open_swap("aft", "candidate-nonce".to_string());
14370 let hello = {
14371 let handler = handler.clone();
14372 let candidate_ctx = swap.candidate_ctx.clone();
14373 std::thread::spawn(move || {
14374 handler
14375 .handle_hello(
14376 candidate_ctx.connection_id,
14377 Some(candidate_ctx.egress.clone()),
14378 hello_frame_with_nonce(
14379 "aft",
14380 PROTOCOL_VERSION,
14381 8,
14382 Some("candidate-nonce"),
14383 ),
14384 )
14385 .unwrap()
14386 })
14387 };
14388
14389 hold.wait();
14390 assert!(swap
14393 .registry
14394 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
14395 .unwrap()
14396 .is_some());
14397 assert!(swap
14398 .registry
14399 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14400 .unwrap()
14401 .is_none());
14402 assert!(swap
14403 .forwarding
14404 .module_endpoint_for_connection(CANDIDATE)
14405 .unwrap()
14406 .is_none());
14407
14408 hold.wait();
14409 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
14410 assert_eq!(
14411 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
14412 FrameType::HelloAck
14413 );
14414 let published = swap
14415 .registry
14416 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14417 .unwrap()
14418 .expect("candidate is published once its HELLO completes");
14419 assert_eq!(published.connection_id, CANDIDATE);
14420 assert!(swap
14421 .forwarding
14422 .module_endpoint_for_connection(CANDIDATE)
14423 .unwrap()
14424 .is_some());
14425 }
14426
14427 fn keyed_total(counters: &Value, key: &str) -> u64 {
14428 counters[key]
14429 .as_object()
14430 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
14431 .unwrap_or(0)
14432 }
14433
14434 #[tokio::test]
14435 async fn replacement_logs_old_end_and_new_admission_once() {
14436 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14437 let mut swap = swap_with_incumbent().await;
14438 swap.handler
14439 .supervisor
14440 .open_swap("aft", "candidate-nonce".to_string());
14441 hello_via_sink(
14442 &swap.handler,
14443 &swap.candidate_ctx,
14444 &mut swap.candidate_rx,
14445 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
14446 )
14447 .await;
14448 cutover(&swap);
14449 swap.handler.cleanup_connection(INCUMBENT).unwrap();
14450
14451 let captured = crate::router::test_log::captured_logs(&logs);
14452 println!("captured replacement registry logs:\n{captured}");
14453 let new_admissions: Vec<_> = captured
14454 .lines()
14455 .filter(|line| {
14456 line.contains("connection_id=40")
14457 && line.contains("swap candidate registered; not routable until cutover")
14458 })
14459 .collect();
14460 assert_eq!(
14461 new_admissions.len(),
14462 1,
14463 "unexpected admission logs: {captured}"
14464 );
14465 assert!(new_admissions[0].contains("module_id=aft"));
14466 assert!(new_admissions[0].contains("connection_id=40"));
14467 assert!(
14468 new_admissions[0].contains("swap candidate registered; not routable until cutover")
14469 );
14470 assert!(new_admissions[0].contains("ready=true"));
14471
14472 let old_admissions: Vec<_> = captured
14473 .lines()
14474 .filter(|line| {
14475 line.contains("module registered module_id=aft ")
14476 && line.contains("connection_id=30")
14477 })
14478 .collect();
14479 assert_eq!(
14480 old_admissions.len(),
14481 1,
14482 "unexpected incumbent admission: {captured}"
14483 );
14484
14485 let old_ends: Vec<_> = captured
14486 .lines()
14487 .filter(|line| {
14488 line.contains("connection_id=30") && line.contains("module registration ended")
14489 })
14490 .collect();
14491 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
14492 assert!(old_ends[0].contains("module_id=aft"));
14493 assert!(old_ends[0].contains("connection_id=30"));
14494 assert!(old_ends[0].contains("reason=replaced"));
14495 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
14496 assert_eq!(
14497 captured
14498 .lines()
14499 .filter(|line| line.contains("module registration promoted"))
14500 .count(),
14501 1,
14502 "unexpected promotion logs: {captured}"
14503 );
14504 }
14505
14506 #[tokio::test]
14512 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
14513 let mut swap = swap_with_incumbent().await;
14514 let handler = swap.handler.clone();
14515
14516 let cotenant = ConnectionId::new(31);
14518 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
14519 let (cotenant_task, cotenant_bind) = relay_route_open(
14520 &handler,
14521 cotenant,
14522 &cotenant_ctx.egress,
14523 &mut swap.incumbent_rx,
14524 100,
14525 "aft",
14526 "swap-cotenant",
14527 )
14528 .await;
14529 handler
14530 .handle_control_frame(
14531 &swap.incumbent_ctx,
14532 route_bind_ack(cotenant_bind.header.corr),
14533 )
14534 .await
14535 .unwrap();
14536 assert!(cotenant_task.await.unwrap().is_empty());
14537 let (cotenant_channel, cotenant_epoch) =
14538 published_route(&cotenant_rx.recv().await.unwrap());
14539
14540 let caller = ConnectionId::new(32);
14542 let (caller_ctx, mut caller_rx) = route_ctx(caller);
14543 let (caller_task, caller_bind) = relay_route_open(
14544 &handler,
14545 caller,
14546 &caller_ctx.egress,
14547 &mut swap.incumbent_rx,
14548 101,
14549 "aft",
14550 "swap-caller",
14551 )
14552 .await;
14553 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
14554
14555 register_candidate(&swap, None);
14556 cutover(&swap);
14557
14558 let ack = handler
14560 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
14561 .await;
14562 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
14563 if module_loop_error.is_some() {
14564 handler.cleanup_connection(INCUMBENT).unwrap();
14567 }
14568
14569 assert!(
14571 cotenant_rx.try_recv().is_err(),
14572 "the co-tenant route on the incumbent was torn down by one late ack: \
14573 {module_loop_error:?}"
14574 );
14575 assert!(matches!(
14576 swap.forwarding
14577 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14578 .unwrap(),
14579 DataRoute::Client(DataRouteState::Bound(_))
14580 ));
14581 assert_eq!(module_loop_error, None);
14582 assert!(swap
14583 .registry
14584 .get_module_by_connection(INCUMBENT)
14585 .unwrap()
14586 .is_some());
14587
14588 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
14590 .await
14591 .expect("the incumbent is told to drop the abandoned binding")
14592 .unwrap()
14593 .frame;
14594 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14595 assert_eq!(goodbye.header.channel, abandoned_channel);
14596 assert_eq!(goodbye.header.epoch, abandoned_epoch);
14597 assert!(swap.incumbent_rx.try_recv().is_err());
14598
14599 let response = caller_task.await.unwrap();
14601 assert_eq!(response.len(), 1);
14602 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14603 assert!(caller_rx.try_recv().is_err());
14604
14605 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
14609 let counters = handler.counters().snapshot();
14610 assert_eq!(
14611 keyed_total(&counters, "route_open_accepted_by_principal"),
14612 1
14613 );
14614 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
14615 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
14616 }
14617
14618 #[tokio::test]
14622 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
14623 let mut swap = swap_with_incumbent().await;
14624 register_candidate(&swap, None);
14625 let incumbent = cutover(&swap);
14626 swap.forwarding
14627 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
14628 .unwrap()
14629 .expect("the incumbent is still registered");
14630
14631 let client = ConnectionId::new(33);
14632 let (client_ctx, mut client_rx) = route_ctx(client);
14633 let route_handler = swap.handler.clone();
14634 let open_ctx = RouteCtx {
14635 connection_id: client,
14636 egress: client_ctx.egress.clone(),
14637 };
14638 let mut route_task = tokio::spawn(async move {
14639 route_handler
14640 .handle_control_frame(
14641 &open_ctx,
14642 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
14643 )
14644 .await
14645 .unwrap()
14646 });
14647 let bind = tokio::select! {
14648 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
14649 response = &mut route_task => {
14650 let response = response.unwrap();
14651 panic!(
14652 "post-cutover route.open was refused instead of relayed to the candidate: {}",
14653 parse_error(&response[0])["code"]
14654 );
14655 }
14656 };
14657 swap.handler
14658 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
14659 .await
14660 .unwrap();
14661 assert!(route_task.await.unwrap().is_empty());
14662 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
14663 match swap
14664 .forwarding
14665 .lookup_data_route(client, channel, epoch)
14666 .unwrap()
14667 {
14668 DataRoute::Client(DataRouteState::Bound(route)) => {
14669 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
14670 }
14671 other => panic!("expected a bound route on the candidate, got {other:?}"),
14672 }
14673 assert!(swap.incumbent_rx.try_recv().is_err());
14674 }
14675
14676 #[tokio::test]
14681 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
14682 let swap = swap_with_incumbent().await;
14683 register_candidate(&swap, Some(false));
14684 let update = Frame::build(
14685 FrameType::Request,
14686 control_flags(),
14687 0,
14688 0,
14689 55,
14690 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14691 provides: manifest("aft", PROTOCOL_VERSION).provides,
14692 capabilities: None,
14693 ready: Some(true),
14694 })
14695 .unwrap(),
14696 )
14697 .unwrap();
14698
14699 let replies = swap
14700 .handler
14701 .handle_control_frame(&swap.candidate_ctx, update)
14702 .await
14703 .unwrap();
14704
14705 assert_eq!(replies.len(), 1);
14706 assert_eq!(
14707 replies[0].header.ty,
14708 FrameType::Response,
14709 "candidate catalog.update was refused: {:?}",
14710 serde_json::from_slice::<Value>(&replies[0].body).ok()
14711 );
14712 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
14713 assert_eq!(
14714 swap.registry
14715 .get_module("aft")
14716 .unwrap()
14717 .unwrap()
14718 .connection_id,
14719 INCUMBENT
14720 );
14721 }
14722 }
14723
14724 mod swap_admission {
14728 use super::*;
14729
14730 const INCUMBENT_NONCE: &str = "incumbent-nonce";
14731 const CANDIDATE_NONCE: &str = "candidate-nonce";
14732
14733 fn handler_with_incumbent(
14734 module_id: &str,
14735 reserved: bool,
14736 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
14737 let registry = Arc::new(Registry::default());
14738 let supervisor = SupervisorHandle::new();
14739 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
14740 if reserved {
14741 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
14742 }
14743 let handler =
14744 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
14745 let incumbent = handler
14746 .handle_control(
14747 ConnectionId::new(1),
14748 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
14749 )
14750 .unwrap();
14751 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
14752 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
14753 (registry, supervisor, handler)
14754 }
14755
14756 #[test]
14762 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
14763 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
14764
14765 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
14766 let replies = handler
14767 .handle_control(
14768 ConnectionId::new(connection),
14769 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
14770 )
14771 .unwrap();
14772 assert_eq!(replies[0].header.ty, FrameType::Error);
14773 assert_eq!(
14774 parse_error(&replies[0])["code"],
14775 "swap_token_invalid",
14776 "nonce {nonce:?}"
14777 );
14778 }
14779 assert!(registry.get_candidate("aft").unwrap().is_none());
14780 assert_eq!(
14781 registry.get_module("aft").unwrap().unwrap().connection_id,
14782 ConnectionId::new(1)
14783 );
14784
14785 let admitted = handler
14788 .handle_control(
14789 ConnectionId::new(4),
14790 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
14791 )
14792 .unwrap();
14793 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
14794 assert_eq!(
14795 registry
14796 .get_candidate("aft")
14797 .unwrap()
14798 .unwrap()
14799 .connection_id,
14800 ConnectionId::new(4)
14801 );
14802 assert_eq!(
14803 registry.get_module("aft").unwrap().unwrap().connection_id,
14804 ConnectionId::new(1),
14805 "the candidate must not take the active slot"
14806 );
14807 let replayed = handler
14808 .handle_control(
14809 ConnectionId::new(5),
14810 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
14811 )
14812 .unwrap();
14813 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
14814
14815 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
14819 let squatter = handler
14820 .handle_control(
14821 ConnectionId::new(6),
14822 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
14823 )
14824 .unwrap();
14825 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
14826 assert!(
14827 registry.get_module("aft").unwrap().is_none(),
14828 "a squatter took the active slot of an id being swapped"
14829 );
14830 }
14831
14832 #[test]
14837 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
14838 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
14839
14840 let replies = handler
14841 .handle_control(
14842 ConnectionId::new(2),
14843 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14844 )
14845 .unwrap();
14846
14847 assert_eq!(
14848 replies[0].header.ty,
14849 FrameType::HelloAck,
14850 "reserved candidate refused: {:?}",
14851 serde_json::from_slice::<Value>(&replies[0].body).ok()
14852 );
14853 assert_eq!(
14854 registry
14855 .get_candidate("vault")
14856 .unwrap()
14857 .unwrap()
14858 .connection_id,
14859 ConnectionId::new(2)
14860 );
14861 }
14862
14863 #[test]
14866 fn without_an_open_swap_the_ordinary_gates_decide() {
14867 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
14868 supervisor.close_swap("vault");
14869
14870 let candidate = handler
14871 .handle_control(
14872 ConnectionId::new(2),
14873 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14874 )
14875 .unwrap();
14876 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
14877 let duplicate = handler
14878 .handle_control(
14879 ConnectionId::new(3),
14880 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
14881 )
14882 .unwrap();
14883 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14884 assert!(registry.get_candidate("vault").unwrap().is_none());
14885 }
14886 }
14887
14888 mod scopes {
14892 mod idle_tests {
14893 use super::*;
14894 include!("scope_idle_tests.rs");
14895 }
14896 mod route_open_timings {
14897 use super::*;
14898 include!("route_open_timing_tests.rs");
14899 }
14900 mod activity {
14901 include!("activity_scope_tests.rs");
14902 }
14903 mod delta {
14904 include!("scope_handler_delta_tests.rs");
14905 }
14906 mod call_modes {
14907 include!("scope_call_mode_tests.rs");
14908 }
14909 #[tokio::test]
14910 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
14911 let rig = rig().await;
14912 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
14913 "s".into(),
14914 1,
14915 "tool".into(),
14916 ))
14917 .unwrap();
14918 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
14919 let (direct, _rx) = wide_ctx(999);
14920 for (ctx, expected) in [
14921 (&rig.owner, FrameType::Response),
14922 (&direct, FrameType::Error),
14923 ] {
14924 let frame =
14925 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
14926 .unwrap();
14927 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
14928 assert_eq!(replies[0].header.ty, expected);
14929 if expected == FrameType::Response {
14930 let reply: subc_control::ScopeServedByReply =
14931 serde_json::from_slice(&replies[0].body).unwrap();
14932 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
14933 } else {
14934 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
14935 }
14936 }
14937 let reply = call(
14938 &rig.handler,
14939 &rig.owner,
14940 &ModuleControlRequestFromModule::ScopeDescribe {
14941 owner: Principal::Reserved {
14942 module_id: OWNER.into(),
14943 },
14944 scope_ref: "s".into(),
14945 },
14946 )
14947 .await;
14948 assert!(matches!(
14949 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
14950 ModuleControlResponseToModule::ScopeDescribe { .. }
14951 ));
14952 }
14953
14954 use subc_protocol::scope::{
14955 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
14956 ScopeStatus,
14957 };
14958
14959 use super::*;
14960
14961 const OWNER: &str = "prefrontal-core";
14962
14963 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
14964 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
14965 }
14966
14967 async fn call(
14968 handler: &ControlHandler,
14969 ctx: &RouteCtx,
14970 request: &ModuleControlRequestFromModule,
14971 ) -> Frame {
14972 let body = serde_json::to_vec(request).unwrap();
14973 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
14974 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
14975 assert_eq!(replies.len(), 1, "{replies:?}");
14976 replies.pop().unwrap()
14977 }
14978
14979 async fn sync(
14980 handler: &ControlHandler,
14981 ctx: &RouteCtx,
14982 generation: u64,
14983 scopes: Vec<ScopeRecord>,
14984 ) -> Result<ModuleControlResponseToModule, String> {
14985 let reply = call(
14986 handler,
14987 ctx,
14988 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
14989 )
14990 .await;
14991 match reply.header.ty {
14992 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
14993 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
14994 }
14995 }
14996
14997 async fn describe(
14998 handler: &ControlHandler,
14999 ctx: &RouteCtx,
15000 owner: &str,
15001 scope_ref: &str,
15002 ) -> ModuleControlResponseToModule {
15003 let reply = call(
15004 handler,
15005 ctx,
15006 &ModuleControlRequestFromModule::ScopeDescribe {
15007 owner: Principal::Reserved {
15008 module_id: owner.to_string(),
15009 },
15010 scope_ref: scope_ref.to_string(),
15011 },
15012 )
15013 .await;
15014 assert_eq!(
15015 reply.header.ty,
15016 FrameType::Response,
15017 "{:?}",
15018 parse_error(&reply)
15019 );
15020 serde_json::from_slice(&reply.body).unwrap()
15021 }
15022
15023 #[tokio::test]
15024 async fn scope_describe_never_echoes_bind_proof_or_hash() {
15025 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15026 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15027 let mut rig = rig().await;
15028 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15029 .await;
15030 let _bound = rig
15031 .bound(
15032 Some(AFT),
15033 PLEXUS,
15034 Some(
15035 rig.selector("s", Some(1))
15036 .with_bind_proof(Some(PROOF.into())),
15037 ),
15038 )
15039 .await;
15040 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
15041 let output = serde_json::to_string(&reply).unwrap();
15042 assert!(output.contains("\"status\":\"live\""));
15043 assert!(!output.contains(PROOF));
15044 assert!(!output.contains(HASH));
15045 }
15046
15047 async fn module(
15049 handler: &ControlHandler,
15050 connection: u64,
15051 module_id: &str,
15052 nonce: Option<&str>,
15053 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15054 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
15055 hello_via_sink(
15056 handler,
15057 &ctx,
15058 &mut rx,
15059 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
15060 )
15061 .await;
15062 (ctx, rx)
15063 }
15064
15065 #[tokio::test]
15068 async fn a_client_connection_cannot_sync_or_describe() {
15069 let handler = ControlHandler::new(Arc::new(Registry::default()));
15070 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
15071 for request in [
15072 ModuleControlRequestFromModule::ScopeSync {
15073 generation: 1,
15074 scopes: vec![head("s", 1)],
15075 },
15076 ModuleControlRequestFromModule::ScopeDescribe {
15077 owner: Principal::Direct,
15078 scope_ref: "s".to_string(),
15079 },
15080 ] {
15081 let reply = call(&handler, &ctx, &request).await;
15082 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
15083 }
15084 assert!(
15085 !handler
15086 .scopes
15087 .read()
15088 .unwrap()
15089 .describe(
15090 &Principal::Reserved {
15091 module_id: OWNER.to_string()
15092 },
15093 "s"
15094 )
15095 .owner_synced
15096 );
15097 }
15098
15099 #[tokio::test]
15102 async fn a_module_without_a_supervised_launch_cannot_sync() {
15103 let handler = ControlHandler::new(Arc::new(Registry::default()));
15104 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
15105 assert_eq!(
15106 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
15107 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15108 );
15109 }
15110
15111 #[tokio::test]
15112 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
15113 let supervisor = SupervisorHandle::new();
15114 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15115 let handler = ControlHandler::new(Arc::new(Registry::default()))
15116 .with_supervisor(supervisor.clone());
15117 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15118 sync(&handler, &incumbent, 1, vec![head("s", 1)])
15119 .await
15120 .expect("the current launch syncs");
15121
15122 supervisor.open_swap(OWNER, "n2".to_string());
15125 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
15126 assert_eq!(
15127 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15128 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15129 );
15130 sync(&handler, &incumbent, 2, vec![head("s", 1)])
15131 .await
15132 .expect("the serving owner syncs during the swap");
15133
15134 supervisor.close_swap(OWNER);
15137 assert_eq!(
15138 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
15139 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15140 );
15141 sync(&handler, &incumbent, 3, vec![head("s", 1)])
15142 .await
15143 .expect("the serving owner syncs after the rollback");
15144 handler.cleanup_connection(candidate.connection_id).unwrap();
15145
15146 supervisor.open_swap(OWNER, "n3".to_string());
15151 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
15152 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
15153 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
15154 .await
15155 .expect("the promoted launch takes authority");
15156 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15157 panic!("unexpected reply {reply:?}");
15158 };
15159 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
15160 assert_eq!(
15161 sync(&handler, &incumbent, 4, Vec::new()).await,
15162 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
15163 );
15164 }
15165
15166 #[tokio::test]
15169 async fn closing_the_authority_connection_frees_sync_authority() {
15170 let supervisor = SupervisorHandle::new();
15171 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15172 let handler = ControlHandler::new(Arc::new(Registry::default()))
15173 .with_supervisor(supervisor.clone());
15174 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
15175 sync(&handler, &first, 10, vec![head("s", 1)])
15176 .await
15177 .unwrap();
15178 handler.cleanup_connection(first.connection_id).unwrap();
15179
15180 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
15181 sync(&handler, &second, 1, vec![head("s", 1)])
15182 .await
15183 .expect("the next connection takes the released authority");
15184 }
15185
15186 #[tokio::test]
15187 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
15188 let supervisor = SupervisorHandle::new();
15189 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
15190 let handler =
15191 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
15192 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
15193 sync(&handler, &first, 10, vec![head("s", 1)])
15194 .await
15195 .unwrap();
15196 handler
15197 .handle_control_frame(
15198 &first,
15199 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
15200 )
15201 .await
15202 .unwrap();
15203 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
15204 sync(&handler, &second, 1, vec![head("s", 1)])
15205 .await
15206 .expect("GOODBYE releases authority even if the old socket remains open");
15207 }
15208
15209 #[tokio::test]
15210 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
15211 let registry = Arc::new(Registry::default());
15212 let supervisor_handle = SupervisorHandle::new();
15213 let supervisor =
15214 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15215 .with_handle(supervisor_handle.clone())
15216 .with_daemon_incarnation("incarnation-7".to_string());
15217 supervisor
15220 .supervise_configured(
15221 ModuleSpec {
15222 module_id: OWNER.to_string(),
15223 program: PathBuf::from("/nonexistent/prefrontal-core"),
15224 args: Vec::new(),
15225 env: Vec::new(),
15226 reserved: false,
15227 reserved_prefixes: Vec::new(),
15228 protocol: ModuleProtocol::Subc,
15229 overlap: Default::default(),
15230 },
15231 false,
15232 )
15233 .unwrap();
15234 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
15235 let handler =
15236 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15237 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
15238
15239 let ModuleControlResponseToModule::ScopeDescribe {
15241 status,
15242 daemon_incarnation,
15243 owner_synced,
15244 owner_configured,
15245 scope,
15246 ..
15247 } = describe(&handler, &reader, OWNER, "s").await
15248 else {
15249 panic!("not a describe reply");
15250 };
15251 assert_eq!(status, ScopeStatus::NotLive);
15252 assert_eq!(daemon_incarnation, "incarnation-7");
15253 assert!(!owner_synced);
15254 assert!(owner_configured);
15255 assert!(scope.is_none());
15256
15257 let ModuleControlResponseToModule::ScopeDescribe {
15260 status,
15261 owner_configured,
15262 ..
15263 } = describe(&handler, &reader, "ghost", "s").await
15264 else {
15265 panic!("not a describe reply");
15266 };
15267 assert_eq!(status, ScopeStatus::NotLive);
15268 assert!(!owner_configured);
15269
15270 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
15272 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
15273 let ModuleControlResponseToModule::ScopeDescribe {
15274 status,
15275 scope_epoch,
15276 owner_synced,
15277 scope,
15278 ..
15279 } = describe(&handler, &reader, OWNER, "s").await
15280 else {
15281 panic!("not a describe reply");
15282 };
15283 assert_eq!(status, ScopeStatus::Live);
15284 assert_eq!(scope_epoch, Some(4));
15285 assert!(owner_synced);
15286 let stamp = scope.expect("a live scope carries its stamp");
15287 assert!(
15288 stamp.owner_authorized,
15289 "prefrontal-core is the default authority"
15290 );
15291 assert_eq!(stamp.kind, ScopeKind::Head);
15292 }
15293
15294 #[tokio::test]
15295 async fn scope_authority_owners_decides_owner_authorized() {
15296 let supervisor = SupervisorHandle::new();
15297 supervisor.set_spawn_nonce("broca", "b1".to_string());
15298 let handler = ControlHandler::new(Arc::new(Registry::default()))
15299 .with_supervisor(supervisor)
15300 .with_scope_authority_owners(vec!["broca".to_string()]);
15301 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
15302 let mut gated = head("s", 1);
15303 gated.attributes.agent_id = Some("agent".to_string());
15304 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
15305 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
15306 describe(&handler, &broca, "broca", "s").await
15307 else {
15308 panic!("not a describe reply");
15309 };
15310 assert!(scope.unwrap().owner_authorized);
15311 }
15312
15313 #[tokio::test]
15317 async fn scope_ops_and_the_scopes_capability_are_advertised() {
15318 let handler = ControlHandler::new(Arc::new(Registry::default()));
15319 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
15320 let ack = hello_via_sink(
15321 &handler,
15322 &ctx,
15323 &mut rx,
15324 hello_frame("m", PROTOCOL_VERSION, 1),
15325 )
15326 .await;
15327 let ack = parse_ack(&ack);
15328 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
15329 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
15330 }
15331 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
15332
15333 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
15334 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15335 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15336 let reply = handler
15337 .handle_control_frame(&client, frame)
15338 .await
15339 .unwrap()
15340 .pop()
15341 .unwrap();
15342 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15343 serde_json::from_slice(&reply.body).unwrap()
15344 else {
15345 panic!("not a server.describe reply");
15346 };
15347 assert!(
15348 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
15349 "{capabilities:?}"
15350 );
15351 }
15352
15353 const PLEXUS: &str = "plexus";
15356 const OTHER: &str = "other";
15357 const AFT: &str = "aft";
15358 const BROCA: &str = "broca";
15359 const MAGIC: &str = "magic-context";
15360
15361 fn nonce(module_id: &str) -> String {
15362 format!("nonce-{module_id}")
15363 }
15364
15365 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15366 let (tx, rx) = mpsc::channel(64);
15367 (
15368 RouteCtx {
15369 connection_id: ConnectionId::new(connection),
15370 egress: FrameSink::new(tx),
15371 },
15372 rx,
15373 )
15374 }
15375
15376 struct Rig {
15380 handler: ControlHandler,
15381 forwarding: Arc<ForwardingTable>,
15382 owner: RouteCtx,
15383 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15384 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
15385 generation: u64,
15386 next_connection: u64,
15387 _supervisor: Supervisor,
15388 }
15389
15390 async fn rig() -> Rig {
15391 rig_with_flow_support(true).await
15392 }
15393
15394 async fn rig_with_flow_support(flow_support: bool) -> Rig {
15395 rig_with_scope_capabilities(flow_support, false).await
15396 }
15397
15398 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
15399 let mut provides = Vec::new();
15400 if flow_support {
15401 provides.push("flow-scopes/v1");
15402 }
15403 if run_support {
15404 provides.push("agent-run-scopes/v1");
15405 }
15406 rig_with_target_capabilities(&provides).await
15407 }
15408
15409 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
15410 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
15411 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15412 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15415 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
15416 Frame::build(
15417 FrameType::Hello,
15418 control_flags(),
15419 0,
15420 0,
15421 connection,
15422 serde_json::to_vec(&body).unwrap(),
15423 )
15424 .unwrap()
15425 }
15426
15427 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
15428 let registry = Arc::new(Registry::default());
15429 let forwarding = Arc::new(ForwardingTable::default());
15430 let supervisor_handle = SupervisorHandle::new();
15431 let supervisor =
15432 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15433 .with_handle(supervisor_handle.clone());
15434 supervisor
15435 .supervise_configured(
15436 ModuleSpec {
15437 module_id: OWNER.to_string(),
15438 program: PathBuf::from("/nonexistent/prefrontal-core"),
15439 args: Vec::new(),
15440 env: Vec::new(),
15441 reserved: false,
15442 reserved_prefixes: Vec::new(),
15443 protocol: ModuleProtocol::Subc,
15444 overlap: Default::default(),
15445 },
15446 false,
15447 )
15448 .unwrap();
15449 for module_id in [OWNER, AFT, BROCA, MAGIC] {
15450 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
15451 }
15452 let handler =
15453 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15454 .with_supervisor(supervisor_handle);
15455 let (owner, mut owner_rx) = wide_ctx(1);
15456 hello_via_sink(
15457 &handler,
15458 &owner,
15459 &mut owner_rx,
15460 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
15461 )
15462 .await;
15463 let mut modules = BTreeMap::new();
15464 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
15465 let (ctx, mut rx) = wide_ctx(connection);
15466 let hello = hello_with_capabilities(module_id, connection, provides);
15467 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
15468 modules.insert(module_id.to_string(), (ctx, rx));
15469 }
15470 Rig {
15471 handler,
15472 forwarding,
15473 owner,
15474 _owner_rx: owner_rx,
15475 modules,
15476 generation: 0,
15477 next_connection: 100,
15478 _supervisor: supervisor,
15479 }
15480 }
15481
15482 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
15483 ScopeCarrier::new(Principal::Reserved {
15484 module_id: module_id.to_string(),
15485 })
15486 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
15487 }
15488
15489 fn session(scope_epoch: u64) -> ScopeRecord {
15492 let mut record = head("s", scope_epoch);
15493 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
15494 record.attributes.agent_id = Some("agent-1".to_string());
15495 record.attributes.delegates = true;
15496 record
15497 }
15498
15499 impl Rig {
15500 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
15501 self.generation += 1;
15502 sync(&self.handler, &self.owner, self.generation, scopes)
15503 .await
15504 .expect("the owner's sync is accepted");
15505 }
15506
15507 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15508 ScopeSelector::new(
15509 Principal::Reserved {
15510 module_id: OWNER.to_string(),
15511 },
15512 scope_ref.to_string(),
15513 scope_epoch,
15514 )
15515 }
15516
15517 fn open_frame(
15518 &mut self,
15519 opener: Option<&str>,
15520 target: &str,
15521 scope: Option<ScopeSelector>,
15522 ) -> (
15523 RouteCtx,
15524 mpsc::Receiver<crate::router::OutboundFrame>,
15525 Frame,
15526 ) {
15527 self.next_connection += 1;
15528 let (ctx, rx) = wide_ctx(self.next_connection);
15529 let root = unique_project_root("scoped-open");
15530 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15531 target: RouteTarget::ToolProvider {
15532 module_id: target.to_string(),
15533 },
15534 identity: BindIdentity::new(
15535 root.path().to_path_buf(),
15536 "unit".to_string(),
15537 "session".to_string(),
15538 ),
15539 consumer_identity: opener.map(|module_id| ConsumerIdentity {
15540 module_id: module_id.to_string(),
15541 launch_nonce: nonce(module_id),
15542 }),
15543 consumer_capabilities: None,
15544 role_versions: None,
15545 admission_facts: None,
15546 scope,
15547 })
15548 .unwrap();
15549 let frame = Frame::build(
15550 FrameType::Request,
15551 control_flags(),
15552 0,
15553 0,
15554 self.next_connection,
15555 body,
15556 )
15557 .unwrap();
15558 (ctx, rx, frame)
15559 }
15560
15561 async fn refused(
15563 &mut self,
15564 opener: Option<&str>,
15565 target: &str,
15566 scope: Option<ScopeSelector>,
15567 ) -> String {
15568 self.refusal_body(opener, target, scope).await["code"]
15569 .as_str()
15570 .unwrap()
15571 .to_string()
15572 }
15573
15574 async fn refusal_body(
15575 &mut self,
15576 opener: Option<&str>,
15577 target: &str,
15578 scope: Option<ScopeSelector>,
15579 ) -> Value {
15580 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
15581 let replies = tokio::time::timeout(
15582 Duration::from_secs(2),
15583 self.handler.handle_control_frame(&ctx, frame),
15584 )
15585 .await
15586 .expect("the open must be refused before waiting for a bind ack")
15587 .unwrap();
15588 assert_eq!(replies.len(), 1, "{replies:?}");
15589 assert_eq!(replies[0].header.ty, FrameType::Error);
15590 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15591 assert!(
15592 module_rx.try_recv().is_err(),
15593 "a refused open relays nothing"
15594 );
15595 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
15596 parse_error(&replies[0])
15597 }
15598
15599 async fn relayed(
15601 &mut self,
15602 opener: Option<&str>,
15603 target: &str,
15604 scope: Option<ScopeSelector>,
15605 ) -> Relayed {
15606 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
15607 let handler = self.handler.clone();
15608 let task_ctx = ctx.clone();
15609 let task = tokio::spawn(async move {
15610 handler
15611 .handle_control_frame(&task_ctx, frame)
15612 .await
15613 .unwrap()
15614 });
15615 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15616 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
15617 .await
15618 .expect("the target receives the relayed route.bind")
15619 .unwrap()
15620 .frame;
15621 Relayed {
15622 target: target.to_string(),
15623 client: ctx,
15624 client_rx: rx,
15625 task,
15626 bind,
15627 }
15628 }
15629
15630 async fn ack(&self, relayed: &Relayed) {
15631 let (module, _) = &self.modules[&relayed.target];
15632 self.handler
15633 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
15634 .await
15635 .unwrap();
15636 }
15637
15638 async fn bound(
15640 &mut self,
15641 opener: Option<&str>,
15642 target: &str,
15643 scope: Option<ScopeSelector>,
15644 ) -> Bound {
15645 let relayed = self.relayed(opener, target, scope).await;
15646 self.ack(&relayed).await;
15647 let Relayed {
15648 target,
15649 client,
15650 mut client_rx,
15651 task,
15652 bind,
15653 } = relayed;
15654 assert!(
15655 task.await.unwrap().is_empty(),
15656 "the open is answered by commit"
15657 );
15658 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
15659 Bound {
15660 target,
15661 client,
15662 client_rx,
15663 channel,
15664 epoch,
15665 bind,
15666 }
15667 }
15668
15669 fn live(&self, route: &Bound) -> bool {
15670 matches!(
15671 self.forwarding
15672 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15673 .unwrap(),
15674 DataRoute::Client(DataRouteState::Bound(_))
15675 )
15676 }
15677 }
15678
15679 struct Relayed {
15680 target: String,
15681 client: RouteCtx,
15682 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15683 task: tokio::task::JoinHandle<Vec<Frame>>,
15684 bind: Frame,
15685 }
15686
15687 struct Bound {
15688 target: String,
15689 client: RouteCtx,
15690 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15691 channel: u16,
15692 epoch: u32,
15693 bind: Frame,
15694 }
15695
15696 impl Bound {
15697 fn closed_reason(&mut self) -> RouteCloseReason {
15700 let mut reason = None;
15701 let mut goodbye = false;
15702 while let Ok(outbound) = self.client_rx.try_recv() {
15703 let frame = outbound.frame;
15704 match frame.header.ty {
15705 FrameType::Goodbye => {
15706 assert_eq!(
15707 (frame.header.channel, frame.header.epoch),
15708 (self.channel, self.epoch)
15709 );
15710 goodbye = true;
15711 }
15712 FrameType::Push => {
15713 let ClientControlPush::RouteClosed {
15714 reason: r,
15715 module_id,
15716 ..
15717 } = serde_json::from_slice(&frame.body).unwrap()
15718 else {
15719 panic!("unexpected push");
15720 };
15721 assert_eq!(module_id, self.target);
15722 reason = Some(r);
15723 }
15724 other => panic!("unexpected frame {other:?}"),
15725 }
15726 }
15727 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
15728 reason.expect("the client is told why the route closed")
15729 }
15730
15731 fn untouched(&mut self) -> bool {
15732 self.client_rx.try_recv().is_err()
15733 }
15734
15735 fn stamp(&self) -> Option<ScopeStamp> {
15736 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
15737 ModuleControlRequest::RouteBind { scope, .. } => scope,
15738 other => panic!("expected a route.bind, got {other:?}"),
15739 }
15740 }
15741 }
15742
15743 #[tokio::test]
15744 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
15745 ) {
15746 let mut rig = rig().await;
15747 let mut record = session(1);
15748 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
15749 record.child_owners = vec![Principal::Reserved {
15750 module_id: MAGIC.to_string(),
15751 }];
15752 rig.sync(vec![record]).await;
15753 let scope = || Some(rig_selector("s", Some(1)));
15754
15755 rig.bound(Some(OWNER), PLEXUS, scope()).await;
15758 rig.bound(Some(AFT), OTHER, scope()).await;
15759 rig.bound(Some(BROCA), PLEXUS, scope()).await;
15760
15761 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
15765 assert_eq!(
15766 rig.refused(opener, target, scope()).await,
15767 error_codes::SCOPE_NOT_CARRIER,
15768 "{opener:?} -> {target}"
15769 );
15770 }
15771 }
15772
15773 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15774 ScopeSelector::new(
15775 Principal::Reserved {
15776 module_id: OWNER.to_string(),
15777 },
15778 scope_ref.to_string(),
15779 scope_epoch,
15780 )
15781 }
15782
15783 #[tokio::test]
15784 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
15785 let mut rig = rig().await;
15786 rig.sync(vec![session(1)]).await;
15787 for opener in [OWNER, AFT] {
15788 assert_eq!(
15789 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
15790 .await,
15791 error_codes::SCOPE_EPOCH_REQUIRED,
15792 "{opener}"
15793 );
15794 }
15795 }
15796
15797 #[tokio::test]
15798 async fn admission_separates_not_synced_not_live_and_ended() {
15799 let mut rig = rig().await;
15800 let code = rig
15802 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15803 .await;
15804 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
15805 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
15806
15807 let ghost = ScopeSelector::new(
15809 Principal::Reserved {
15810 module_id: "ghost".to_string(),
15811 },
15812 "s".to_string(),
15813 Some(1),
15814 );
15815 assert_eq!(
15816 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
15817 error_codes::SCOPE_NOT_LIVE
15818 );
15819
15820 rig.sync(vec![session(2)]).await;
15821 assert_eq!(
15822 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
15823 .await,
15824 error_codes::SCOPE_NOT_LIVE
15825 );
15826 for epoch in [1, 3] {
15827 assert_eq!(
15828 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
15829 .await,
15830 error_codes::SCOPE_ENDED,
15831 "epoch {epoch}"
15832 );
15833 }
15834 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
15836 .await;
15837 }
15838
15839 #[tokio::test]
15840 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
15841 let mut rig = rig().await;
15842 rig.sync(vec![session(1)]).await;
15843 let route = rig
15844 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15845 .await;
15846 let stamp = route.stamp().expect("a scoped bind carries the stamp");
15847 assert_eq!(stamp.scope_ref, "s");
15848 assert_eq!(stamp.scope_epoch, 1);
15849 assert_eq!(stamp.kind, ScopeKind::Head);
15850 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
15851 assert!(stamp.attributes.delegates);
15852 assert!(stamp.owner_authorized);
15853 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15854 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
15855
15856 let (broca, mut broca_rx) = wide_ctx(50);
15859 hello_via_sink(
15860 &rig.handler,
15861 &broca,
15862 &mut broca_rx,
15863 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
15864 )
15865 .await;
15866 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
15867 .await
15868 .unwrap();
15869 let own = ScopeSelector::new(
15870 Principal::Reserved {
15871 module_id: BROCA.to_string(),
15872 },
15873 "b".to_string(),
15874 Some(1),
15875 );
15876 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
15877 assert!(!route.stamp().unwrap().owner_authorized);
15878 }
15879
15880 #[tokio::test]
15881 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
15882 let mut rig = rig().await;
15883 let mut record = head("s", 1);
15884 record.carriers = vec![carrier(AFT, None)];
15885 let flow_id = "Flow:run-7/step_2!~";
15886 record.attributes.flow_id = Some(flow_id.to_string());
15887 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
15888 .await
15889 .unwrap();
15890 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15891 panic!("not a sync reply");
15892 };
15893 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
15894 let route = rig
15895 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15896 .await;
15897 assert!(rig.live(&route), "the stamped bind committed");
15898 let stamp = route.stamp().expect("a flow scope carries a stamp");
15899 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
15900 assert_eq!(stamp.attributes.agent_id, None);
15901 assert!(!stamp.attributes.delegates);
15902 assert!(stamp.owner_authorized);
15903 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15904 assert_eq!(unscoped.stamp(), None);
15905 }
15906
15907 #[tokio::test]
15908 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
15909 let mut rig = rig_with_flow_support(false).await;
15910 let mut record = session(1);
15911 record.attributes.flow_id = Some("flow:7".to_string());
15912 rig.sync(vec![record]).await;
15913 for opener in [OWNER, AFT] {
15914 let body = rig
15915 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15916 .await;
15917 assert_eq!(body["code"], "target_flow_unsupported");
15918 let message = body["message"].as_str().unwrap();
15919 for required in [PLEXUS, "flow-scopes/v1"] {
15920 assert!(message.contains(required), "{message}");
15921 }
15922 }
15923 }
15924
15925 #[tokio::test]
15926 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
15927 let mut rig = rig_with_flow_support(true).await;
15928 let mut record = session(1);
15929 record.attributes.flow_id = Some("flow:7".to_string());
15930 rig.sync(vec![record]).await;
15931 for opener in [OWNER, AFT] {
15932 let route = rig
15933 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15934 .await;
15935 assert!(rig.live(&route));
15936 assert_eq!(
15937 route.stamp().unwrap().attributes.flow_id.as_deref(),
15938 Some("flow:7")
15939 );
15940 }
15941 }
15942
15943 #[tokio::test]
15944 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
15945 use std::future::Future;
15946
15947 let mut rig = rig_with_flow_support(true).await;
15948 let mut record = session(1);
15949 record.attributes.flow_id = Some("flow:7".to_string());
15950 rig.sync(vec![record]).await;
15951 let (client, mut client_rx, frame) =
15952 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
15953 for _ in 0..64 {
15956 client.egress.try_send(route_bind_ack(1)).unwrap();
15957 }
15958 let handler = rig.handler.clone();
15959 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15960 std::future::poll_fn(|cx| {
15961 assert!(open.as_mut().poll(cx).is_pending());
15962 std::task::Poll::Ready(())
15963 })
15964 .await;
15965 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15966
15967 let old_connection = rig.modules[PLEXUS].0.connection_id;
15968 rig.handler.cleanup_connection(old_connection).unwrap();
15969 let (replacement, mut replacement_rx) = wide_ctx(200);
15970 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
15971 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15972 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15973 let hello = Frame::build(
15974 FrameType::Hello,
15975 control_flags(),
15976 0,
15977 0,
15978 200,
15979 serde_json::to_vec(&body).unwrap(),
15980 )
15981 .unwrap();
15982 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
15983
15984 client_rx.try_recv().unwrap();
15985 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15986 .await
15987 .expect("the replacement is refused without waiting for a bind ack")
15988 .unwrap();
15989 assert_eq!(replies.len(), 1);
15990 let body = parse_error(&replies[0]);
15991 assert_eq!(body["code"], "target_flow_unsupported");
15992 for required in [PLEXUS, "flow-scopes/v1"] {
15993 assert!(body["message"].as_str().unwrap().contains(required));
15994 }
15995 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
15996 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15997 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15998 assert!(rig
15999 .handler
16000 .registry
16001 .get_module_by_connection(replacement.connection_id)
16002 .unwrap()
16003 .is_some());
16004 }
16005
16006 #[tokio::test]
16007 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
16008 use std::future::Future;
16009 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16010 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16011 const NEXT_HASH: &str =
16012 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16013 let mut rig = rig().await;
16014 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
16015 record.carriers.push(ScopeCarrier::new(Principal::Direct));
16016 rig.sync(vec![record.clone()]).await;
16017 let (client, mut rx, frame) = rig.open_frame(
16018 None,
16019 PLEXUS,
16020 Some(
16021 rig.selector("s", Some(1))
16022 .with_bind_proof(Some(PROOF.into())),
16023 ),
16024 );
16025 for _ in 0..64 {
16028 client.egress.try_send(route_bind_ack(1)).unwrap();
16029 }
16030 let handler = rig.handler.clone();
16031 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16032 std::future::poll_fn(|cx| {
16033 assert!(open.as_mut().poll(cx).is_pending());
16034 std::task::Poll::Ready(())
16035 })
16036 .await;
16037 record.bind_proof_sha256 = Some(NEXT_HASH.into());
16038 rig.sync(vec![record]).await;
16039 rx.try_recv().unwrap();
16040 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16041 .await
16042 .unwrap()
16043 .unwrap();
16044 assert_eq!(
16045 parse_error(&replies[0])["code"],
16046 error_codes::SCOPE_BIND_PROOF_MISMATCH
16047 );
16048 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
16049 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
16050 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16051 }
16052
16053 #[tokio::test]
16054 async fn scope_bind_proof_reserved_openers_ignore_proof() {
16055 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16056 let mut rig = rig().await;
16057 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16058 .await;
16059 for opener in [OWNER, AFT] {
16060 for proof in [None, Some("malformed+proof=")] {
16061 let route = rig
16062 .bound(
16063 Some(opener),
16064 PLEXUS,
16065 Some(
16066 rig.selector("s", Some(1))
16067 .with_bind_proof(proof.map(str::to_string)),
16068 ),
16069 )
16070 .await;
16071 assert!(rig.live(&route));
16072 }
16073 }
16074 }
16075
16076 #[tokio::test]
16077 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
16078 use std::future::Future;
16079 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16080 const NEXT_HASH: &str =
16081 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
16082 let mut rig = rig().await;
16083 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16084 .await;
16085 let (client, mut rx, frame) =
16086 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
16087 for _ in 0..64 {
16088 client.egress.try_send(route_bind_ack(1)).unwrap();
16089 }
16090 let handler = rig.handler.clone();
16091 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
16092 std::future::poll_fn(|cx| {
16093 assert!(open.as_mut().poll(cx).is_pending());
16094 std::task::Poll::Ready(())
16095 })
16096 .await;
16097 rig.sync(vec![
16098 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
16099 ])
16100 .await;
16101 rx.try_recv().unwrap();
16102 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16103 let bind = std::future::poll_fn(|cx| {
16104 assert!(open.as_mut().poll(cx).is_pending());
16105 module_rx.poll_recv(cx)
16106 })
16107 .await
16108 .unwrap()
16109 .frame;
16110 handler
16111 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
16112 .await
16113 .unwrap();
16114 let replies = tokio::time::timeout(Duration::from_secs(2), open)
16115 .await
16116 .unwrap()
16117 .unwrap();
16118 assert_eq!(replies.len(), 1);
16119 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
16122 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16123 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
16124 let route = rig
16125 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
16126 .await;
16127 assert!(rig.live(&route));
16128 }
16129
16130 #[tokio::test]
16131 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
16132 {
16133 let mut rig = rig_with_flow_support(false).await;
16134 rig.sync(vec![session(1)]).await;
16135 let route = rig
16136 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16137 .await;
16138 assert!(rig.live(&route));
16139 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
16140 let mut record = session(1);
16141 record.attributes.flow_id = Some("flow:7".to_string());
16142 rig.sync(vec![record]).await;
16143 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
16144 assert!(rig.live(&unscoped));
16145 assert_eq!(unscoped.stamp(), None);
16146 }
16147
16148 #[tokio::test]
16149 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
16150 let mut rig = rig().await;
16151 let mut record = session(1);
16152 record.attributes.flow_id = Some("flow:7".to_string());
16153 rig.sync(vec![record.clone()]).await;
16154 let before = rig.forwarding.published_scope_tag(OWNER, "s");
16155 let mut owner_route = rig
16156 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16157 .await;
16158 let mut carrier_route = rig
16159 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
16160 .await;
16161 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
16162 rig.sync(vec![record.clone()]).await;
16163 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16164 assert!(rig.live(&owner_route) && owner_route.untouched());
16165 assert!(rig.live(&carrier_route) && carrier_route.untouched());
16166
16167 record.attributes.flow_id = Some("flow:8".to_string());
16168 rig.sync(vec![record]).await;
16169 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
16170 let before = before.unwrap();
16171 assert_eq!(after.scope_epoch, before.scope_epoch);
16172 assert!(after.version > before.version);
16173 for route in [&mut owner_route, &mut carrier_route] {
16174 assert!(!rig.live(route));
16175 assert_eq!(
16176 route.closed_reason(),
16177 RouteCloseReason::ScopeDelegationChanged
16178 );
16179 }
16180 assert!(rig.live(&unscoped) && unscoped.untouched());
16181 for target in [PLEXUS, OTHER] {
16184 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
16185 let goodbye = module_rx
16186 .try_recv()
16187 .expect("the provider sees the drain")
16188 .frame;
16189 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16190 assert!(module_rx.try_recv().is_err());
16191 }
16192 let rebound = rig
16193 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16194 .await;
16195 assert_eq!(
16196 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
16197 Some("flow:8")
16198 );
16199 }
16200
16201 #[tokio::test]
16206 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
16207 let mut rig = rig().await;
16208 rig.sync(vec![session(1)]).await;
16209 let mut cotenant = rig
16210 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
16211 .await;
16212
16213 let mut changed = session(1);
16214 changed.child_owners.push(Principal::Reserved {
16215 module_id: MAGIC.to_string(),
16216 });
16217 let mut ended = None;
16218 for (code, next) in [
16219 (error_codes::SCOPE_CHANGED, vec![changed]),
16220 (error_codes::SCOPE_ENDED, Vec::new()),
16221 ] {
16222 let relayed = rig
16223 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16224 .await;
16225 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
16226 ended = Some(next.is_empty());
16227 rig.sync(next).await;
16228 rig.ack(&relayed).await;
16229 let replies = relayed.task.await.unwrap();
16230 assert_eq!(replies.len(), 1, "{replies:?}");
16231 assert_eq!(parse_error(&replies[0])["code"], code);
16232 assert_eq!(
16233 subc_protocol::error_codes::is_retryable_route_open(code),
16234 code == error_codes::SCOPE_CHANGED
16235 );
16236 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
16237 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16239 let mut goodbyes = Vec::new();
16242 while let Ok(outbound) = plexus_rx.try_recv() {
16243 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
16244 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
16245 }
16246 assert!(
16247 goodbyes.contains(&(bind_channel, bind_epoch)),
16248 "{goodbyes:?}"
16249 );
16250 assert!(rig
16251 .handler
16252 .registry
16253 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
16254 .unwrap()
16255 .is_some());
16256 }
16257 assert_eq!(ended, Some(true));
16258 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
16261 }
16262
16263 #[tokio::test]
16266 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
16267 struct Case {
16268 name: &'static str,
16269 change: fn(&mut ScopeRecord),
16270 closed: [Option<RouteCloseReason>; 4],
16273 }
16274 use RouteCloseReason::*;
16275 let cases = [
16276 Case {
16277 name: "a carrier entry removed",
16278 change: |r| {
16279 r.carriers.retain(|c| {
16280 c.principal
16281 != Principal::Reserved {
16282 module_id: AFT.to_string(),
16283 }
16284 })
16285 },
16286 closed: [None, Some(ScopeCarrierRemoved), None, None],
16287 },
16288 Case {
16289 name: "a target removed from a carrier",
16290 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
16291 closed: [None, None, None, Some(ScopeCarrierRemoved)],
16292 },
16293 Case {
16294 name: "a bare carrier narrowed to targets",
16295 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
16296 closed: [None, Some(ScopeCarrierRemoved), None, None],
16297 },
16298 Case {
16299 name: "delegates turned off",
16300 change: |r| r.attributes.delegates = false,
16301 closed: [Some(ScopeDelegationChanged); 4],
16302 },
16303 Case {
16304 name: "agent_id changed",
16305 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
16306 closed: [Some(ScopeDelegationChanged); 4],
16307 },
16308 Case {
16309 name: "a carrier added, child owners changed, the record re-sent",
16310 change: |r| {
16311 r.carriers.push(carrier(MAGIC, None));
16312 r.child_owners.push(Principal::Reserved {
16313 module_id: MAGIC.to_string(),
16314 });
16315 },
16316 closed: [None; 4],
16317 },
16318 Case {
16319 name: "a target added",
16320 change: |r| {
16321 r.carriers[1]
16322 .targets
16323 .as_mut()
16324 .unwrap()
16325 .push("third".to_string())
16326 },
16327 closed: [None; 4],
16328 },
16329 Case {
16330 name: "delegates turned on",
16331 change: |r| r.attributes.delegates = true,
16332 closed: [None; 4],
16333 },
16334 ];
16335 for case in cases {
16336 let mut rig = rig().await;
16337 rig.sync(vec![session(1)]).await;
16338 let scope = || Some(rig_selector("s", Some(1)));
16339 let mut routes = [
16340 rig.bound(Some(OWNER), PLEXUS, scope()).await,
16341 rig.bound(Some(AFT), PLEXUS, scope()).await,
16342 rig.bound(Some(BROCA), PLEXUS, scope()).await,
16343 rig.bound(Some(BROCA), OTHER, scope()).await,
16344 ];
16345 let mut record = session(1);
16346 (case.change)(&mut record);
16347 rig.sync(vec![record]).await;
16348 for (index, expected) in case.closed.iter().enumerate() {
16349 let route = &mut routes[index];
16350 match expected {
16351 Some(reason) => {
16352 assert!(!rig.live(route), "{}: route {index} still live", case.name);
16353 assert_eq!(
16354 route.closed_reason(),
16355 *reason,
16356 "{}: route {index}",
16357 case.name
16358 );
16359 }
16360 None => {
16361 assert!(rig.live(route), "{}: route {index} closed", case.name);
16362 assert!(
16363 route.untouched(),
16364 "{}: route {index} was told something",
16365 case.name
16366 );
16367 }
16368 }
16369 }
16370 }
16371 }
16372
16373 #[tokio::test]
16374 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
16375 for next in [Vec::new(), vec![session(2)]] {
16377 let mut rig = rig().await;
16378 rig.sync(vec![session(1)]).await;
16379 let mut route = rig
16380 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16381 .await;
16382 rig.sync(next).await;
16383 assert!(!rig.live(&route));
16384 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
16385 }
16386
16387 let mut rig = rig().await;
16390 let mut child = session(1);
16391 child.scope_ref = "child".to_string();
16392 child.kind = ScopeKind::Worker;
16393 child.parent = Some(ScopeParent::new(
16394 Principal::Reserved {
16395 module_id: OWNER.to_string(),
16396 },
16397 "s".to_string(),
16398 1,
16399 ));
16400 rig.sync(vec![session(1), child.clone()]).await;
16401 let mut child_route = rig
16402 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
16403 .await;
16404 assert_eq!(
16405 child_route.stamp().unwrap().parent_state,
16406 Some(ParentState::Linked)
16407 );
16408 rig.sync(vec![child]).await;
16409 assert!(!rig.live(&child_route));
16410 assert_eq!(
16411 child_route.closed_reason(),
16412 RouteCloseReason::ScopeParentEnded
16413 );
16414 }
16415
16416 #[tokio::test]
16417 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
16418 ) {
16419 let mut rig = rig().await;
16420 rig.sync(vec![session(1)]).await;
16421 let mut route = rig
16422 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16423 .await;
16424 let before = rig.forwarding.published_scope_tag(OWNER, "s");
16425 rig.sync(vec![session(1)]).await;
16426 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16427 assert!(rig.live(&route) && route.untouched());
16428
16429 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
16433 .forwarding
16434 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16435 .unwrap()
16436 else {
16437 panic!("the route is bound");
16438 };
16439 binding.flow.acquire_tagged(9, false).await.unwrap();
16440 let mut widened = session(1);
16441 widened.carriers.push(carrier(MAGIC, None));
16442 rig.sync(vec![widened]).await;
16443 assert!(rig.live(&route) && route.untouched());
16444 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16445 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
16446 assert_eq!(binding.flow.in_flight(), 1);
16449 binding
16450 .flow
16451 .acquire_tagged(10, false)
16452 .await
16453 .expect("the flow is still open");
16454 }
16455
16456 #[tokio::test]
16459 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
16460 let mut rig = rig().await;
16461 rig.sync(vec![session(1)]).await;
16462 let mut on_incumbent = rig
16463 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16464 .await;
16465
16466 let (candidate, _candidate_rx) = wide_ctx(9);
16469 let registration = rig
16470 .handler
16471 .registry
16472 .register_candidate_with_control_ops(
16473 manifest(PLEXUS, PROTOCOL_VERSION),
16474 PROTOCOL_VERSION,
16475 candidate.connection_id,
16476 module_baseline_control_ops(),
16477 )
16478 .unwrap();
16479 rig.forwarding
16480 .register_candidate_module_connection(
16481 candidate.connection_id,
16482 PLEXUS.to_string(),
16483 PROTOCOL_VERSION,
16484 manifest_concurrency(®istration.manifest),
16485 candidate.egress.clone(),
16486 )
16487 .unwrap();
16488 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
16489 rig.handler
16490 .registry
16491 .promote_candidate(PLEXUS)
16492 .unwrap()
16493 .unwrap();
16494 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
16495
16496 rig.sync(Vec::new()).await;
16497 assert!(!rig.live(&on_incumbent));
16498 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
16499 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16500 let goodbye = incumbent_rx
16501 .try_recv()
16502 .expect("the superseded endpoint is told")
16503 .frame;
16504 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16505 }
16506 }
16507}
16508
16509#[cfg(test)]
16510mod concurrency_default_exposure_tests {
16511 use super::*;
16512
16513 fn hello_body(role_json: &str) -> Vec<u8> {
16514 format!(
16515 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
16516 )
16517 .into_bytes()
16518 }
16519
16520 fn manifest_from(body: &[u8]) -> ModuleManifest {
16521 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
16522 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
16523 .expect("manifest parses")
16524 }
16525
16526 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
16527
16528 #[test]
16529 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
16530 let body = hello_body(&format!(
16531 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
16532 ));
16533 let manifest = manifest_from(&body);
16534 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16537 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
16538 }
16539
16540 #[test]
16541 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
16542 let body = hello_body(&format!(
16543 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
16544 ));
16545 let manifest = manifest_from(&body);
16546 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16547 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16548 }
16549
16550 #[test]
16551 fn non_management_roles_are_never_reported() {
16552 let body = hello_body(
16553 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
16554 );
16555 let manifest = manifest_from(&body);
16556 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16557 }
16558}