1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_event_declarations,
25 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
26 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
27 },
28 scope::{
29 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
30 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
31 SCOPE_SYNC_OP,
32 },
33 session::{
34 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
35 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
36 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
37 ROLE_VERSIONS_FIELD,
38 },
39 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
40 Priority, RouteTarget, PROTOCOL_VERSION,
41};
42use tokio::time::{timeout_at, Instant};
43use tracing::{debug, info, warn};
44
45use crate::{
46 caller_identity::CallerIdentity,
47 capability_requirements::{
48 log_duplicate_claim_events, log_requirement_events, CapabilityRequirementEvaluator,
49 CapabilityVerdict, DuplicateClaimSource, RegisteredModule, RequirementStatus,
50 RuntimeModule,
51 },
52 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
53 forwarding::{
54 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
55 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
56 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
57 },
58 observability::{
59 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
60 },
61 privileged_prompt::{prompt_text, PrivilegedOperation},
62 provenance::{
63 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
64 },
65 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
66 router::{RouteCtx, RouterError},
67 scopes::{HelloLaunchNonces, ScopeTable},
68 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
69 stderr_tail::{CaptureState, TailEntry},
70 supervise::{
71 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
72 SupervisorHandle, SwapHelloAdmission,
73 },
74 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
75};
76
77pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
83
84const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
85const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
86const CAP_PING_PONG: &str = "ping_pong_v1";
87const CAP_SESSION_ATTACH: &str = "session_attach_v1";
88const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
89
90const SUBC_CONTROL_OPS: &[&str] = &[
91 ops::ACTIVITY_RECORD,
92 ops::ACTIVITY_QUERY,
93 ops::ACTIVITY_SUBSCRIBE,
94 ops::SERVER_DESCRIBE,
95 ops::CATALOG_LIST,
96 ops::ROUTE_OPEN,
97 ops::ROUTE_POLL,
98 ops::ROUTE_CLOSING,
99 ops::ROUTE_CLOSED,
100 ops::SUPERVISOR_LIST,
101 ops::SUPERVISOR_RESTART,
102 ops::SUPERVISOR_SWAP,
103 ops::SUPERVISOR_RELOAD,
104 ops::SUPERVISOR_RESCAN,
105 ops::SUPERVISOR_RELEASE_RESERVED,
106 ops::SUPERVISOR_SET_ENABLED,
107 ops::SUPERVISOR_HEALTH_PROBE,
108 ops::SUPERVISOR_HEALTH,
109 ops::SUPERVISOR_STDERR_TAIL,
110 ops::SUPERVISOR_TERMINALS,
111 ops::SUPERVISOR_ROUTES,
112 ops::SUPERVISOR_PROVENANCE,
113 ops::SUPERVISOR_SPAWN_SNAPSHOT,
114 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
115];
116
117const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
118 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
119 "supervisor.live_roots",
120 SCOPE_SYNC_OP,
121 SCOPE_APPLY_OP,
122 SCOPE_DESCRIBE_OP,
123 "operator.confirm",
124];
125
126const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
130
131const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
132
133pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
141
142pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
157
158pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
176
177const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
178const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
179
180fn reload_verdict(
181 configured: &Path,
182 spawned_from: Option<&Path>,
183 image: subc_control::RunningImageAgreement,
184) -> PendingReloadVerdict {
185 let path = match spawned_from {
186 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
187 ReloadPathAgreement::Match
188 }
189 Some(spawned_from) => ReloadPathAgreement::Mismatch {
190 configured: configured.to_path_buf(),
191 spawned_from: spawned_from.to_path_buf(),
192 },
193 None => ReloadPathAgreement::Unavailable {
194 reason: if matches!(
195 image,
196 subc_control::RunningImageAgreement::Unavailable {
197 reason: subc_control::RunningImageUnavailableReason::NotRunning
198 }
199 ) {
200 ReloadPathUnavailableReason::NotRunning
201 } else {
202 ReloadPathUnavailableReason::SpawnedPathUnavailable
203 },
204 },
205 };
206 PendingReloadVerdict {
207 path,
208 image,
209 spec: None,
210 }
211}
212
213fn launch_spec_agreement(
214 configured: &crate::supervise::ModuleSpec,
215 running: bool,
216 spawned: Option<&crate::supervise::LaunchSpecDigest>,
217) -> subc_control::LaunchSpecAgreement {
218 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
219 if !running {
220 return Agreement::Unavailable {
221 reason: Reason::NotRunning,
222 };
223 }
224 match spawned {
225 None => Agreement::Unavailable {
226 reason: Reason::SpawnedSpecUnavailable,
227 },
228 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
229 Agreement::Match
230 }
231 Some(_) => Agreement::Mismatch,
232 }
233}
234
235fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
236 if configured == spawned_from {
237 return true;
238 }
239 #[cfg(windows)]
240 {
241 match (
247 std::fs::canonicalize(configured),
248 std::fs::canonicalize(spawned_from),
249 ) {
250 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
251 _ => false,
252 }
253 }
254 #[cfg(not(windows))]
255 {
256 false
257 }
258}
259
260#[derive(Clone)]
261struct DaemonProvenanceFacts {
262 build: DaemonBuildProvenance,
263 pid: Option<u32>,
264 started_at_ms: Option<u64>,
265 start_clock: Option<crate::clock::StartClock>,
266 executable_path: Option<PathBuf>,
267 executable_identity: Option<SpawnedFileIdentity>,
268 process_start_time: Option<u64>,
269 probe: ExecutableIdentityProbe,
270}
271
272impl Default for DaemonProvenanceFacts {
273 fn default() -> Self {
274 Self {
275 build: DaemonBuildProvenance {
276 build_git_sha: None,
277 build_lock_digest: None,
278 },
279 pid: None,
280 started_at_ms: None,
281 start_clock: None,
282 executable_path: None,
283 executable_identity: None,
284 process_start_time: None,
285 probe: ExecutableIdentityProbe::default(),
286 }
287 }
288}
289
290#[derive(Debug, Clone)]
291struct SupervisorRescanContext {
292 supervisor: Supervisor,
293 config_path: PathBuf,
294 configured_port: Option<u16>,
295 storage_config: Option<crate::daemon_config::StorageConfig>,
296 admission_facts_carrier_module_id: Option<String>,
297 admission_facts_targets: Option<Vec<String>>,
298 scope_authority_owners: Vec<String>,
299 privileged_operator_prompt: PrivilegedOperatorPrompt,
300}
301
302const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
323 "reloading",
324 "supervisor_not_live",
325 "registration_not_active",
326 "no_forwarding_connection",
327 "relay_send_failed",
328];
329
330#[derive(Clone)]
332pub struct ControlHandler {
333 registry: Arc<Registry>,
334 activity: Option<crate::activity::ActivityLedger>,
335 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
336 forwarding: Arc<ForwardingTable>,
337 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
338 supervisor: SupervisorHandle,
339 subc_capabilities: Arc<[String]>,
340 route_bind_relay_timeout: Duration,
344 route_bind_relay_timeouts: BTreeMap<String, Duration>,
348 route_bind_breakers: RouteBindBreakers,
351 route_bind_concurrency: RouteBindConcurrency,
354 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
359 route_bind_breaker_threshold: u32,
361 route_bind_breaker_cooldown: Duration,
363 health_probe_timeout: Duration,
364 storage_config: Option<crate::daemon_config::StorageConfig>,
367 machine_id: Option<crate::machine_id::MachineId>,
371 admission_facts_carrier_module_id: Option<String>,
372 admission_facts_targets: Option<Vec<String>>,
373 scopes: Arc<RwLock<ScopeTable>>,
377 scope_deadline_added: Arc<tokio::sync::Notify>,
381 scope_authority_owners: Vec<String>,
384 privileged_operator_prompt: PrivilegedOperatorPrompt,
386 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
389 #[cfg(test)]
390 caller_lookup: Option<TestCallerLookup>,
391 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
394 rescan: Option<SupervisorRescanContext>,
395 connected_clients: ConnectedClients,
396 counters: DaemonCounters,
397 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
398 daemon_provenance: DaemonProvenanceFacts,
399 #[cfg(test)]
400 control_dispatch_delay: Option<Duration>,
401 #[cfg(test)]
402 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
403 #[cfg(test)]
407 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
408 #[cfg(test)]
414 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
415}
416
417impl fmt::Debug for ControlHandler {
418 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
419 f.debug_struct("ControlHandler")
420 .field("registry", &self.registry)
421 .field("forwarding", &self.forwarding)
422 .field("process_liveness", &self.process_liveness.is_some())
423 .field("supervisor", &self.supervisor)
424 .field("subc_capabilities", &self.subc_capabilities)
425 .finish()
426 }
427}
428
429struct RouteOpenRequest {
430 target: RouteTarget,
431 identity: BindIdentity,
432 consumer_identity: Option<ConsumerIdentity>,
433 consumer_capabilities: Option<Vec<String>>,
434 role_versions: Option<BTreeMap<String, String>>,
435 admission_facts: Option<serde_json::Value>,
436 scope: Option<ScopeSelector>,
437}
438
439#[derive(Clone, Copy)]
440enum RouteOpenPhase {
441 Admission,
442 ProjectRoot,
443 EgressReservation,
444 BindSend,
445 BindAckWait,
446}
447
448struct RouteOpenTimings {
452 module_id: String,
453 principal: String,
454 admission: Duration,
455 project_root: Duration,
456 egress_reservation: Duration,
457 bind_send: Duration,
458 bind_ack_wait: Duration,
459 phase: Option<RouteOpenPhase>,
460 phase_started_at: Instant,
461}
462
463impl RouteOpenTimings {
464 fn new() -> Self {
465 Self {
466 module_id: String::new(),
467 principal: "unresolved".to_string(),
469 admission: Duration::ZERO,
470 project_root: Duration::ZERO,
471 egress_reservation: Duration::ZERO,
472 bind_send: Duration::ZERO,
473 bind_ack_wait: Duration::ZERO,
474 phase: Some(RouteOpenPhase::Admission),
475 phase_started_at: Instant::now(),
476 }
477 }
478
479 fn finish(&mut self) {
480 let elapsed = self.phase_started_at.elapsed();
481 match self.phase.take() {
482 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
483 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
484 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
485 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
486 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
487 None => {}
488 }
489 }
490
491 fn enter(&mut self, phase: RouteOpenPhase) {
492 self.finish();
493 self.phase = Some(phase);
494 self.phase_started_at = Instant::now();
495 }
496}
497
498struct RouteBindReservationGuard {
499 forwarding: Arc<ForwardingTable>,
500 endpoint: ModuleEndpointId,
501 relay_corr: u64,
502 armed: bool,
503}
504
505struct ModuleControlRpcGuard {
506 forwarding: Arc<ForwardingTable>,
507 endpoint: ModuleEndpointId,
508 corr: u64,
509 armed: bool,
510}
511
512impl ModuleControlRpcGuard {
513 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
514 Self {
515 forwarding,
516 endpoint,
517 corr,
518 armed: true,
519 }
520 }
521
522 fn disarm(&mut self) {
523 self.armed = false;
524 }
525}
526
527impl Drop for ModuleControlRpcGuard {
528 fn drop(&mut self) {
529 if self.armed {
530 let _ = self
531 .forwarding
532 .cancel_module_control_rpc(self.endpoint, self.corr);
533 }
534 }
535}
536
537impl RouteBindReservationGuard {
538 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
539 Self {
540 forwarding,
541 endpoint,
542 relay_corr,
543 armed: true,
544 }
545 }
546
547 fn release_and_disarm(&mut self) {
548 if !self.armed {
549 return;
550 }
551 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
552 self.endpoint,
553 self.relay_corr,
554 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
555 ) {
556 send_goodbye_target_best_effort(
557 &self.forwarding.counters(),
558 &target,
559 "canceled route.bind",
560 );
561 }
562 self.armed = false;
563 }
564
565 fn disarm(&mut self) {
566 self.armed = false;
567 }
568}
569
570impl Drop for RouteBindReservationGuard {
571 fn drop(&mut self) {
572 self.release_and_disarm();
573 }
574}
575
576#[derive(Debug, Clone, Default)]
606pub(crate) struct RouteBindBreakers {
607 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
608}
609
610#[derive(Debug, Clone, Default)]
611pub(crate) struct RouteBindConcurrency {
612 modules: Arc<Mutex<HashMap<String, usize>>>,
613}
614
615struct RouteBindConcurrencyGuard {
616 concurrency: RouteBindConcurrency,
617 module_id: String,
618}
619
620impl RouteBindConcurrency {
621 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
624 let mut modules = self
625 .modules
626 .lock()
627 .expect("route.bind concurrency mutex poisoned");
628 let in_flight = modules.entry(module_id.to_string()).or_default();
629 if *in_flight >= limit {
630 return Err(*in_flight);
631 }
632 *in_flight += 1;
633 Ok(RouteBindConcurrencyGuard {
634 concurrency: self.clone(),
635 module_id: module_id.to_string(),
636 })
637 }
638}
639
640impl Drop for RouteBindConcurrencyGuard {
641 fn drop(&mut self) {
642 let mut modules = self
643 .concurrency
644 .modules
645 .lock()
646 .expect("route.bind concurrency mutex poisoned");
647 let remove = {
648 let in_flight = modules
649 .get_mut(&self.module_id)
650 .expect("admitted route.bind has a concurrency entry");
651 *in_flight -= 1;
652 *in_flight == 0
653 };
654 if remove {
655 modules.remove(&self.module_id);
656 }
657 }
658}
659
660#[derive(Debug, Default)]
661struct ModuleBreakerState {
662 consecutive_timeouts: u32,
664 cooldown_until: Option<Instant>,
667 probe_in_flight: Option<Arc<()>>,
672}
673
674enum RouteBindAdmission<'a> {
676 Admitted {
677 guard: RouteBindBreakerGuard<'a>,
678 probe: bool,
681 },
682 Refused {
683 consecutive_timeouts: u32,
684 retry_in: Duration,
688 probe_in_flight: bool,
689 },
690}
691
692struct RouteBindBreakerGuard<'a> {
699 breakers: RouteBindBreakers,
700 module_id: &'a str,
701 probe_token: Option<Arc<()>>,
702 settled: bool,
703}
704
705impl RouteBindBreakerGuard<'_> {
706 fn record_accepted(&mut self) -> bool {
710 self.settled = true;
711 self.breakers.record_accepted(self.module_id)
712 }
713
714 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
717 self.settled = true;
718 self.breakers.record_timeout(
719 self.module_id,
720 self.probe_token.as_ref(),
721 threshold,
722 cooldown,
723 )
724 }
725
726 fn record_inconclusive(&mut self) {
735 self.settled = true;
736 self.breakers
737 .record_inconclusive(self.module_id, self.probe_token.as_ref());
738 }
739}
740
741impl Drop for RouteBindBreakerGuard<'_> {
742 fn drop(&mut self) {
743 if !self.settled {
744 self.breakers
745 .record_inconclusive(self.module_id, self.probe_token.as_ref());
746 }
747 }
748}
749
750struct BreakerOpened {
754 consecutive_timeouts: u32,
755 reopened_after_probe: bool,
758}
759
760impl RouteBindBreakers {
761 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
762 self.modules
763 .lock()
764 .expect("route.bind breaker mutex poisoned")
765 }
766
767 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
770 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
771 probe: probe_token.is_some(),
772 guard: RouteBindBreakerGuard {
773 breakers: self.clone(),
774 module_id,
775 probe_token,
776 settled: false,
777 },
778 };
779
780 let mut modules = self.lock();
781 let Some(state) = modules.get_mut(module_id) else {
782 return admitted(None);
783 };
784 let Some(cooldown_until) = state.cooldown_until else {
785 return admitted(None);
786 };
787 if state.probe_in_flight.is_some() {
788 return RouteBindAdmission::Refused {
789 consecutive_timeouts: state.consecutive_timeouts,
790 retry_in: Duration::ZERO,
791 probe_in_flight: true,
792 };
793 }
794 let now = Instant::now();
795 if now < cooldown_until {
796 return RouteBindAdmission::Refused {
797 consecutive_timeouts: state.consecutive_timeouts,
798 retry_in: cooldown_until - now,
799 probe_in_flight: false,
800 };
801 }
802 let token = Arc::new(());
803 state.probe_in_flight = Some(Arc::clone(&token));
804 admitted(Some(token))
805 }
806
807 fn record_accepted(&self, module_id: &str) -> bool {
808 self.lock()
809 .remove(module_id)
810 .is_some_and(|state| state.cooldown_until.is_some())
811 }
812
813 fn record_timeout(
814 &self,
815 module_id: &str,
816 probe_token: Option<&Arc<()>>,
817 threshold: u32,
818 cooldown: Duration,
819 ) -> Option<BreakerOpened> {
820 let mut modules = self.lock();
821 let state = modules.entry(module_id.to_string()).or_default();
822 let was_open = state.cooldown_until.is_some();
823 let was_probe = Self::owns_probe(state, probe_token);
824 if was_probe {
825 state.probe_in_flight = None;
826 }
827 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
828 if state.consecutive_timeouts < threshold {
829 return None;
830 }
831 state.cooldown_until = Some(Instant::now() + cooldown);
832 Some(BreakerOpened {
833 consecutive_timeouts: state.consecutive_timeouts,
834 reopened_after_probe: was_open && was_probe,
835 })
836 }
837
838 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
839 state
843 .probe_in_flight
844 .as_ref()
845 .zip(token)
846 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
847 }
848
849 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
850 if let Some(state) = self.lock().get_mut(module_id) {
851 if Self::owns_probe(state, probe_token) {
852 state.probe_in_flight = None;
853 }
854 }
855 }
856
857 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
872 self.lock()
873 .remove(module_id)
874 .map(|state| state.consecutive_timeouts)
875 .filter(|discarded| *discarded > 0)
876 }
877
878 fn open_snapshot(&self) -> Option<serde_json::Value> {
885 let now = Instant::now();
886 let modules = self.lock();
887 let open = modules
888 .iter()
889 .filter_map(|(module_id, state)| {
890 let cooldown_until = state.cooldown_until?;
891 Some((
892 module_id.clone(),
893 serde_json::json!({
894 "consecutive_timeouts": state.consecutive_timeouts,
895 "cooldown_remaining_ms":
896 cooldown_until.saturating_duration_since(now).as_millis() as u64,
897 "probe_in_flight": state.probe_in_flight.is_some(),
898 }),
899 ))
900 })
901 .collect::<serde_json::Map<String, serde_json::Value>>();
902 (!open.is_empty()).then_some(serde_json::Value::Object(open))
903 }
904}
905
906impl ControlHandler {
907 pub fn new(registry: Arc<Registry>) -> Self {
908 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
909 }
910
911 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
912 let counters = forwarding.counters();
913 let route_bind_breakers = forwarding.route_bind_breakers();
917 let route_bind_concurrency = forwarding.route_bind_concurrency();
918 let route_outages = forwarding.route_outages();
919 Self {
920 registry,
921 activity: None,
922 activity_streams: Arc::new(Mutex::new(HashMap::new())),
923 forwarding,
924 process_liveness: None,
925 supervisor: SupervisorHandle::new(),
926 subc_capabilities: Arc::from([
927 CAP_MANIFEST_REGISTRATION.to_string(),
928 CAP_CHANNEL_LIFECYCLE.to_string(),
929 CAP_PING_PONG.to_string(),
930 CAP_SESSION_ATTACH.to_string(),
931 CAP_ADMISSION_FACTS_RELAY.to_string(),
932 CAP_SCOPES_V1.to_string(),
933 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
934 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
935 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
936 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
937 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
938 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
939 ]),
940 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
941 route_bind_relay_timeouts: BTreeMap::new(),
942 route_bind_breakers,
943 route_bind_concurrency,
944 route_outages,
945 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
946 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
947 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
948 storage_config: None,
949 machine_id: None,
950 admission_facts_carrier_module_id: None,
951 admission_facts_targets: None,
952 scopes: Arc::new(RwLock::new(ScopeTable::new(
953 crate::daemon_config::default_scope_authority_owners(),
954 ))),
955 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
956 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
957 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
958 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
959 #[cfg(test)]
960 caller_lookup: None,
961 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
962 rescan: None,
963 connected_clients: ConnectedClients::new(),
964 counters,
965 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
966 daemon_provenance: DaemonProvenanceFacts::default(),
967 #[cfg(test)]
968 control_dispatch_delay: None,
969 #[cfg(test)]
970 provenance_probe_override: None,
971 #[cfg(test)]
972 route_bind_relay_sent_hold: None,
973 #[cfg(test)]
974 hello_registry_to_forwarding_hold: None,
975 }
976 }
977
978 pub fn with_storage_config(
981 mut self,
982 storage_config: Option<crate::daemon_config::StorageConfig>,
983 ) -> Self {
984 self.storage_config = storage_config;
985 self
986 }
987
988 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
991 self.activity = Some(crate::activity::ActivityLedger::open(
992 run_dir.join("activity.jsonl"),
993 incarnation,
994 ));
995 self.supervisor.set_activity(self.activity.clone());
996 self
997 }
998
999 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1002 self.machine_id = machine_id;
1003 self
1004 }
1005
1006 pub fn with_admission_facts_config(
1010 mut self,
1011 carrier_module_id: Option<String>,
1012 targets: Option<Vec<String>>,
1013 ) -> Self {
1014 self.admission_facts_carrier_module_id = carrier_module_id;
1015 self.admission_facts_targets = targets;
1016 self
1017 }
1018
1019 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1023 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1024 self.scope_authority_owners = owners;
1025 self
1026 }
1027
1028 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1030 self.privileged_operator_prompt = mode;
1031 self
1032 }
1033
1034 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1037 self.route_bind_relay_timeout = timeout;
1038 self
1039 }
1040
1041 pub fn with_route_bind_relay_timeouts(
1047 mut self,
1048 timeouts: impl IntoIterator<Item = (String, Duration)>,
1049 ) -> Self {
1050 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1051 self
1052 }
1053
1054 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1060 self.route_bind_relay_timeouts
1061 .get(module_id)
1062 .copied()
1063 .unwrap_or(self.route_bind_relay_timeout)
1064 }
1065
1066 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1074 self.route_bind_breaker_threshold = threshold.max(1);
1075 self.route_bind_breaker_cooldown = cooldown;
1076 self
1077 }
1078
1079 #[cfg(test)]
1080 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1081 self.health_probe_timeout = timeout;
1082 self
1083 }
1084
1085 #[cfg(test)]
1086 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1087 self.control_dispatch_delay = Some(delay);
1088 self
1089 }
1090
1091 #[cfg(test)]
1095 pub(crate) fn with_route_bind_relay_sent_hold(
1096 mut self,
1097 hold: Arc<tokio::sync::Barrier>,
1098 ) -> Self {
1099 self.route_bind_relay_sent_hold = Some(hold);
1100 self
1101 }
1102
1103 #[cfg(test)]
1108 pub(crate) fn with_hello_registry_to_forwarding_hold(
1109 mut self,
1110 hold: Arc<std::sync::Barrier>,
1111 ) -> Self {
1112 self.hello_registry_to_forwarding_hold = Some(hold);
1113 self
1114 }
1115
1116 #[cfg(test)]
1117 fn hold_between_registry_and_forwarding(&self) {
1118 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1119 hold.wait();
1120 hold.wait();
1121 }
1122 }
1123
1124 pub fn with_process_liveness(
1125 mut self,
1126 process_liveness: Arc<dyn ModuleProcessLiveness>,
1127 ) -> Self {
1128 self.process_liveness = Some(process_liveness);
1129 self
1130 }
1131
1132 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1133 self.supervisor = supervisor;
1134 self.supervisor.set_activity(self.activity.clone());
1135 self
1136 }
1137
1138 pub fn with_daemon_provenance(
1139 mut self,
1140 pid: u32,
1141 started_at_ms: u64,
1142 executable_path: Option<PathBuf>,
1143 build_git_sha: Option<String>,
1144 build_lock_digest: Option<String>,
1145 ) -> Self {
1146 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1147 let process_start_time = process_start_time(pid);
1148 self.daemon_provenance = DaemonProvenanceFacts {
1149 build: DaemonBuildProvenance {
1150 build_git_sha,
1151 build_lock_digest,
1152 },
1153 pid: Some(pid),
1154 started_at_ms: Some(started_at_ms),
1155 start_clock: None,
1156 executable_path,
1157 executable_identity,
1158 process_start_time,
1159 probe: ExecutableIdentityProbe::default(),
1160 };
1161 self
1162 }
1163
1164 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1165 self.daemon_provenance.start_clock = Some(clock);
1166 self
1167 }
1168
1169 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1171 self.scopes
1172 .write()
1173 .unwrap_or_else(|poisoned| poisoned.into_inner())
1174 .set_wall_clock(Arc::new(clock));
1175 self
1176 }
1177
1178 #[cfg(test)]
1179 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1180 self.provenance_probe_override = Some(result);
1181 self
1182 }
1183
1184 pub fn with_capability_config(
1188 self,
1189 modules: impl IntoIterator<Item = (String, bool)>,
1190 reserved_capabilities: BTreeMap<String, String>,
1191 ) -> Self {
1192 self.capability_evaluator
1193 .configure(modules, reserved_capabilities);
1194 self
1195 }
1196
1197 pub fn with_supervisor_rescan(
1198 mut self,
1199 supervisor: Supervisor,
1200 config_path: impl Into<PathBuf>,
1201 configured_port: Option<u16>,
1202 ) -> Self {
1203 self.rescan = Some(SupervisorRescanContext {
1204 supervisor,
1205 config_path: config_path.into(),
1206 configured_port,
1207 storage_config: self.storage_config.clone(),
1208 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1209 admission_facts_targets: self.admission_facts_targets.clone(),
1210 scope_authority_owners: self.scope_authority_owners.clone(),
1211 privileged_operator_prompt: self.privileged_operator_prompt,
1212 });
1213 self
1214 }
1215
1216 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1217 self.connected_clients = connected_clients;
1218 self
1219 }
1220
1221 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1222 Arc::clone(&self.forwarding)
1223 }
1224
1225 pub(crate) fn counters(&self) -> DaemonCounters {
1226 self.counters.clone()
1227 }
1228
1229 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1232 tokio::spawn(async move {
1233 loop {
1234 self.capability_evaluator
1235 .wait_for_change_or_deadline()
1236 .await;
1237 self.refresh_capability_requirements();
1238 }
1239 });
1240 }
1241
1242 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1248 tokio::spawn(async move {
1249 loop {
1250 let has_deadlines = self
1251 .scopes
1252 .read()
1253 .unwrap_or_else(|poisoned| poisoned.into_inner())
1254 .has_deadlines();
1255 if !has_deadlines {
1256 self.scope_deadline_added.notified().await;
1257 continue;
1258 }
1259 tokio::time::sleep(Duration::from_secs(1)).await;
1260 if let Err(error) = self.sweep_expired_scopes() {
1261 warn!(%error, "scope expiry sweep failed");
1262 }
1263 }
1264 });
1265 }
1266
1267 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1268 let mut table = self
1269 .scopes
1270 .write()
1271 .unwrap_or_else(|poisoned| poisoned.into_inner());
1272 let swept = table.sweep_expired();
1273 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1274 drop(table);
1275 self.close_scope_drained_routes(drained);
1276 Ok(())
1277 }
1278
1279 fn publish_scope_changes(
1280 &self,
1281 changes: &[crate::scopes::ScopeTagChange],
1282 expired: &[crate::scopes::ScopeExpired],
1283 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1284 if changes.is_empty() {
1285 return Ok(Vec::new());
1286 }
1287 let drained = self
1288 .forwarding
1289 .publish_scope_changes(changes)
1290 .map_err(RouterError::Forwarding)?;
1291 let mut counts = HashMap::new();
1292 for route in &drained {
1293 *counts
1294 .entry((
1295 route.scope.owner.as_str(),
1296 route.scope.scope_ref.as_str(),
1297 route.scope.tag.scope_epoch,
1298 ))
1299 .or_insert(0usize) += 1;
1300 }
1301 for scope in expired {
1302 let routes_closed = counts
1303 .get(&(
1304 scope.owner.as_str(),
1305 scope.scope_ref.as_str(),
1306 scope.scope_epoch,
1307 ))
1308 .copied()
1309 .unwrap_or(0);
1310 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1311 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1312 routes_closed, "scope expired");
1313 }
1314 Ok(drained)
1315 }
1316
1317 fn runtime_capability_snapshot(
1318 &self,
1319 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1320 let runtime = self
1321 .supervisor
1322 .list()
1323 .into_iter()
1324 .map(|module| {
1325 let status = module.status().map_err(|err| {
1326 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1327 })?;
1328 Ok(RuntimeModule {
1329 module_id: status.module_id,
1330 state: status.state,
1331 enabled: status.enabled,
1332 })
1333 })
1334 .collect::<Result<Vec<_>, RouterError>>()?;
1335 let registrations = self.registry.list_registered_modules().map_err(|err| {
1341 RouterError::backend(
1342 0,
1343 0,
1344 format!("failed to list capability registrations: {err}"),
1345 )
1346 })?;
1347 let registrations = registrations
1348 .into_iter()
1349 .map(|registration| RegisteredModule {
1350 module_id: registration.manifest.module_id,
1351 module_version: registration.manifest.module_version,
1352 capabilities: registration.manifest.capabilities,
1353 })
1354 .collect();
1355 Ok((runtime, registrations))
1356 }
1357
1358 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1365 let cached_registration = RegisteredModule {
1366 module_id: registration.manifest.module_id.clone(),
1367 module_version: registration.manifest.module_version.clone(),
1368 capabilities: registration.manifest.capabilities.clone(),
1369 };
1370 if self.capability_evaluator.record_hello(&cached_registration) {
1371 warn!(
1372 module_id = %cached_registration.module_id,
1373 "capability claims drifted from the cached manifest"
1374 );
1375 }
1376 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1377 self.enforce_capability_denies();
1378 }
1379 self.refresh_capability_requirements();
1380 }
1381
1382 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1386 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1387 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1388 self.supervisor.set_swap_promotion_observer(observer);
1389 }
1390
1391 pub fn refresh_capability_requirements(&self) {
1392 match self.runtime_capability_snapshot() {
1393 Ok((runtime, registrations)) => {
1394 log_requirement_events(
1395 self.capability_evaluator
1396 .evaluate_now(&runtime, ®istrations),
1397 );
1398 }
1399 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1400 }
1401 }
1402
1403 fn enforce_capability_denies(&self) {
1407 let registrations = match self.registry.list_registered_modules() {
1408 Ok(snapshot) => snapshot,
1409 Err(err) => {
1410 warn!(error = %err, "failed to read registrations for capability deny census");
1411 return;
1412 }
1413 };
1414 let manifests = registrations
1415 .into_iter()
1416 .map(|registration| {
1417 (
1418 registration.manifest.module_id.clone(),
1419 registration.manifest,
1420 )
1421 })
1422 .collect::<BTreeMap<_, _>>();
1423 let census = match self.forwarding.route_census(None) {
1424 Ok(census) => census,
1425 Err(err) => {
1426 warn!(error = %err, "failed to read route census for capability deny enforcement");
1427 return;
1428 }
1429 };
1430
1431 for (target_module_id, routes) in census {
1432 let Some(target_manifest) = manifests.get(&target_module_id) else {
1433 continue;
1434 };
1435 let mut closed_routes = Vec::new();
1436 let mut module_goodbyes = Vec::new();
1437 for route in routes {
1438 let Principal::Reserved {
1439 module_id: opening_module_id,
1440 } = &route.principal
1441 else {
1442 continue;
1443 };
1444 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1445 continue;
1446 };
1447 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1448 continue;
1449 };
1450
1451 match self.forwarding.release_client_route(
1452 route.goodbye_target.connection_id,
1453 route.goodbye_target.channel,
1454 route.goodbye_target.epoch,
1455 ) {
1456 Ok(RouteRelease::Removed(module_goodbye)) => {
1457 warn!(
1458 opening_module_id,
1459 target_module_id,
1460 capability,
1461 "force-closing route because an attested capability deny edge now matches"
1462 );
1463 closed_routes.push(route);
1464 module_goodbyes.push(module_goodbye);
1465 }
1466 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1467 Err(err) => warn!(
1468 opening_module_id,
1469 target_module_id,
1470 capability,
1471 error = %err,
1472 "failed to force-close capability-denied route"
1473 ),
1474 }
1475 }
1476
1477 if closed_routes.is_empty() {
1478 continue;
1479 }
1480 send_route_control_pushes(
1481 &self.forwarding,
1482 closed_routes,
1483 ClientControlPush::RouteClosed {
1484 module_id: target_module_id,
1485 channels: Vec::new(),
1486 reason: RouteCloseReason::CapabilityDenied,
1487 drained: false,
1488 abandoned: 0,
1489 excluded_subscriptions: 0,
1490 terminal: Some(false),
1491 },
1492 );
1493 self.emit_route_goodbyes(module_goodbyes);
1494 }
1495 }
1496
1497 fn not_ready_reason(
1503 &self,
1504 registration: &crate::registry::ModuleRegistration,
1505 ) -> Option<NotReadyReason> {
1506 if !registration.ready {
1507 return Some(NotReadyReason {
1508 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1509 capability: None,
1510 });
1511 }
1512 self.first_unprovided_required_capability(registration)
1513 .map(|capability| NotReadyReason {
1514 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1515 capability: Some(capability),
1516 })
1517 }
1518
1519 fn first_unprovided_required_capability(
1537 &self,
1538 registration: &crate::registry::ModuleRegistration,
1539 ) -> Option<String> {
1540 let required = registration
1541 .manifest
1542 .capabilities
1543 .iter()
1544 .flat_map(|declarations| declarations.requires.iter())
1545 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1546 .map(|requirement| requirement.capability.as_str())
1547 .collect::<BTreeSet<_>>();
1548 if required.is_empty() {
1549 return None;
1550 }
1551 let module_id = registration.manifest.module_id.as_str();
1552 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1553 if required
1554 .iter()
1555 .any(|capability| verdict(capability).is_none())
1556 {
1557 self.refresh_capability_requirements();
1558 }
1559 required
1560 .into_iter()
1561 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1562 .map(str::to_string)
1563 }
1564
1565 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1566 self.capability_evaluator
1567 .statuses()
1568 .into_iter()
1569 .map(capability_requirement_status)
1570 .collect()
1571 }
1572
1573 fn deregister_connection(
1580 &self,
1581 connection_id: ConnectionId,
1582 reason: RegistrationEndReason,
1583 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1584 self.registry
1585 .deregister_connection_with_reason(connection_id, reason)
1586 }
1587
1588 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1589 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1590 return None;
1591 }
1592 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1593 parse_client_control_request(&frame.body)
1594 else {
1595 return None;
1596 };
1597 Some(target_module_id(&target).to_string())
1598 }
1599
1600 pub(crate) fn route_open_capacity_refusal(
1601 &self,
1602 ctx: &RouteCtx,
1603 frame: &Frame,
1604 target_module_id: &str,
1605 in_flight: usize,
1606 limit: usize,
1607 ) -> Result<Frame, RouterError> {
1608 self.route_open_admission_refusal_frame(
1609 ctx,
1610 frame,
1611 target_module_id,
1612 "open_admission_full",
1613 (in_flight, limit),
1614 format!(
1615 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1616 ),
1617 )
1618 }
1619
1620 fn route_open_target_capacity_refusal(
1621 &self,
1622 ctx: &RouteCtx,
1623 frame: &Frame,
1624 target_module_id: &str,
1625 in_flight: usize,
1626 ) -> Result<Frame, RouterError> {
1627 self.route_open_admission_refusal_frame(
1628 ctx,
1629 frame,
1630 target_module_id,
1631 "target_binds_full",
1632 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1633 format!(
1634 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1635 ),
1636 )
1637 }
1638
1639 fn route_open_admission_refusal_frame(
1647 &self,
1648 ctx: &RouteCtx,
1649 frame: &Frame,
1650 target_module_id: &str,
1651 reason: &'static str,
1652 (in_flight, limit): (usize, usize),
1653 message: impl Into<String>,
1654 ) -> Result<Frame, RouterError> {
1655 let code = error_codes::TARGET_UNAVAILABLE;
1656 self.counters.increment_route_open_refused(code);
1657 info!(
1658 target: "control",
1659 code,
1660 reason,
1661 module_id = ?target_module_id,
1662 connection_id = ctx.connection_id.get(),
1663 in_flight,
1664 limit,
1665 "route.open refused"
1666 );
1667 control_error_frame(frame, code, message.into())
1668 }
1669
1670 #[cfg(test)]
1676 pub fn handle_control(
1677 &self,
1678 connection_id: ConnectionId,
1679 frame: Frame,
1680 ) -> Result<Vec<Frame>, RouterError> {
1681 match frame.header.ty {
1682 FrameType::Ping => Ok(vec![pong(&frame)?]),
1683 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1684 FrameType::Goodbye => self.handle_goodbye(connection_id),
1685 ty => Ok(vec![control_error_frame(
1686 &frame,
1687 "unsupported_control_frame",
1688 format!("unsupported channel-0 frame {ty:?}"),
1689 )?]),
1690 }
1691 }
1692
1693 pub async fn handle_control_frame(
1694 &self,
1695 ctx: &RouteCtx,
1696 frame: Frame,
1697 ) -> Result<Vec<Frame>, RouterError> {
1698 self.handle_control_frame_timed(ctx, frame, None).await
1699 }
1700
1701 pub(crate) fn record_caller_endpoints(
1702 &self,
1703 connection: ConnectionId,
1704 endpoints: Option<(SocketAddr, SocketAddr)>,
1705 ) {
1706 let mut callers = self
1707 .caller_endpoints
1708 .lock()
1709 .unwrap_or_else(|p| p.into_inner());
1710 callers.remove(&connection);
1711 if let Some(endpoints) = endpoints {
1712 callers.insert(connection, endpoints);
1713 }
1714 }
1715
1716 #[cfg(test)]
1717 pub(crate) fn with_caller_lookup_for_test(
1718 mut self,
1719 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1720 ) -> Self {
1721 self.caller_lookup = Some(Arc::new(lookup));
1722 self
1723 }
1724
1725 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1726 let endpoints = self
1727 .caller_endpoints
1728 .lock()
1729 .unwrap_or_else(|p| p.into_inner())
1730 .get(&connection)
1731 .copied();
1732 #[cfg(test)]
1733 let lookup = self.caller_lookup.clone();
1734 tokio::task::spawn_blocking(move || {
1735 #[cfg(test)]
1736 if let Some(lookup) = lookup {
1737 return lookup(endpoints);
1738 }
1739 endpoints
1740 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1741 .unwrap_or_default()
1742 })
1743 .await
1744 .unwrap_or_default()
1745 }
1746
1747 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1748 frame.header.channel == 0
1749 && frame.header.ty == FrameType::Request
1750 && self
1751 .forwarding
1752 .module_endpoint_for_connection(ctx.connection_id)
1753 .is_ok_and(|endpoint| endpoint.is_none())
1754 && parse_client_control_request(&frame.body)
1755 .ok()
1756 .is_some_and(|request| privileged_operation(&request).is_some())
1757 }
1758
1759 pub(crate) async fn handle_control_frame_timed(
1760 &self,
1761 ctx: &RouteCtx,
1762 frame: Frame,
1763 dispatch_started_at: Option<StdInstant>,
1764 ) -> Result<Vec<Frame>, RouterError> {
1765 match frame.header.ty {
1766 FrameType::Ping => Ok(vec![pong(&frame)?]),
1767 FrameType::Hello => {
1768 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1769 }
1770 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1771 FrameType::Cancel => {
1772 if self
1775 .forwarding
1776 .operator_confirms()
1777 .cancel(ctx.connection_id, frame.header.corr)
1778 || self
1779 .supervisor
1780 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1781 || self.cancel_activity_subscription(ctx, &frame)?
1782 {
1783 Ok(Vec::new())
1784 } else {
1785 Ok(vec![control_error_frame(
1786 &frame,
1787 "unknown_subscription",
1788 "no subscription has this correlation id",
1789 )?])
1790 }
1791 }
1792 FrameType::Request => {
1793 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1796 if op
1797 .as_ref()
1798 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1799 {
1800 return self.handle_activity_record(ctx, frame).await;
1801 }
1802 if op.as_ref().is_some_and(|probe| {
1803 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1804 }) {
1805 return self.handle_activity_read(ctx, frame).await;
1806 }
1807 if op
1808 .as_ref()
1809 .is_some_and(|probe| probe.op == "operator.confirm")
1810 {
1811 return self.handle_operator_confirm(ctx, frame);
1812 }
1813 if self
1814 .forwarding
1815 .module_endpoint_for_connection(ctx.connection_id)
1816 .map_err(RouterError::Forwarding)?
1817 .is_some()
1818 {
1819 if !is_known_module_request_op(&frame.body) {
1820 return Ok(vec![control_error_frame(
1821 &frame,
1822 "unsupported_control_frame",
1823 "module-originated channel-0 REQUEST is not supported",
1824 )?]);
1825 }
1826 let request = match parse_module_control_request_from_module(&frame.body) {
1827 Ok(request) => request,
1828 Err((err, ControlRequestBodyError::UnknownOp)) => {
1829 return Ok(vec![control_error_frame(
1830 &frame,
1831 "unsupported_control_frame",
1832 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1833 )?])
1834 }
1835 Err((err, ControlRequestBodyError::InvalidBody)) => {
1836 return Ok(vec![control_error_frame(
1837 &frame,
1838 "invalid_control_body",
1839 format!("malformed module control body: {err}"),
1840 )?])
1841 }
1842 };
1843 let op = module_control_request_op(&request);
1844 let corr = frame.header.corr;
1845 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1846 let result =
1847 self.handle_module_control_request(ctx.connection_id, frame, request);
1848 log_slow_control_dispatch(
1849 dispatch_started_at,
1850 op,
1851 ctx.connection_id,
1852 corr,
1853 None,
1854 );
1855 return result;
1856 }
1857
1858 if is_known_module_request_op(&frame.body) {
1859 return Ok(vec![control_error_frame(
1860 &frame,
1861 "not_registered",
1862 "catalog.update requires an active module registration owned by this connection",
1863 )?]);
1864 }
1865
1866 let request = match parse_client_control_request(&frame.body) {
1867 Ok(request) => request,
1868 Err((err, ControlRequestBodyError::UnknownOp)) => {
1869 return Ok(vec![control_error_frame(
1870 &frame,
1871 "unknown_control_op",
1872 format!("unknown client control op: {err}"),
1873 )?])
1874 }
1875 Err((err, ControlRequestBodyError::InvalidBody)) => {
1876 return Ok(vec![control_error_frame(
1877 &frame,
1878 "invalid_control_body",
1879 format!("malformed client control body: {err}"),
1880 )?])
1881 }
1882 };
1883 let op = client_control_request_op(&request);
1884 let corr = frame.header.corr;
1885 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1886 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
1887 #[cfg(test)]
1888 if let Some(delay) = self.control_dispatch_delay {
1889 tokio::time::sleep(delay).await;
1890 }
1891 let result = self
1892 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
1893 .await;
1894 if let Some(timings) = route_open_timings.as_mut() {
1895 timings.finish();
1896 }
1897 log_slow_control_dispatch(
1898 dispatch_started_at,
1899 op,
1900 ctx.connection_id,
1901 corr,
1902 route_open_timings.as_ref(),
1903 );
1904 result
1905 }
1906 FrameType::Push => {
1907 let Some(endpoint) = self
1908 .forwarding
1909 .module_endpoint_for_connection(ctx.connection_id)
1910 .map_err(RouterError::Forwarding)?
1911 else {
1912 return Ok(vec![control_error_frame(
1913 &frame,
1914 "unsupported_control_frame",
1915 "client-originated channel-0 PUSH is not supported",
1916 )?]);
1917 };
1918 self.handle_status_update(endpoint, frame)
1919 }
1920 FrameType::Response | FrameType::Error
1921 if self
1922 .forwarding
1923 .module_endpoint_for_connection(ctx.connection_id)
1924 .map_err(RouterError::Forwarding)?
1925 .is_some() =>
1926 {
1927 self.handle_module_relay_response(ctx.connection_id, frame)
1928 }
1929 ty => Ok(vec![control_error_frame(
1930 &frame,
1931 "unsupported_control_frame",
1932 format!("unsupported channel-0 frame {ty:?}"),
1933 )?]),
1934 }
1935 }
1936
1937 pub fn cleanup_connection(
1938 &self,
1939 connection_id: ConnectionId,
1940 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1941 self.caller_endpoints
1942 .lock()
1943 .unwrap_or_else(|p| p.into_inner())
1944 .remove(&connection_id);
1945 self.forwarding
1946 .operator_confirms()
1947 .connection_closed(connection_id);
1948 self.cleanup_connection_with_end_reason(
1949 connection_id,
1950 RegistrationEndReason::ConnectionClosed,
1951 )
1952 }
1953
1954 fn cleanup_connection_with_end_reason(
1955 &self,
1956 connection_id: ConnectionId,
1957 requested_reason: RegistrationEndReason,
1958 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1959 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
1960 self.registry
1961 .get_module_by_connection(connection_id)?
1962 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
1963 .and_then(|module| module.registration_end_reason().ok().flatten())
1964 .unwrap_or(requested_reason)
1965 } else {
1966 requested_reason
1967 };
1968 let crash_closed = self
1969 .registry
1970 .get_module_by_connection(connection_id)?
1971 .and_then(|registration| {
1972 self.forwarding
1973 .module_endpoint_for_connection(connection_id)
1974 .ok()
1975 .flatten()
1976 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
1977 .map(|routes| (registration.manifest.module_id, routes))
1978 });
1979 let crash_closed = crash_closed.map(|(module_id, routes)| {
1980 let terminal = match self.supervisor.get(&module_id) {
1981 None => false,
1982 Some(module) => match module.will_recover_after_connection_loss() {
1983 Ok(will_recover) => !will_recover,
1984 Err(err) => {
1985 warn!(
1986 %module_id,
1987 error = %err,
1988 "failed to read crash recovery verdict; reporting non-terminal conservatively"
1989 );
1990 false
1991 }
1992 },
1993 };
1994 let (reason, drained) = match self
1998 .forwarding
1999 .daemon_drain_result_for_connection(connection_id)
2000 {
2001 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2002 Ok(None) => (RouteCloseReason::Crash, false),
2003 Err(err) => {
2004 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2005 (RouteCloseReason::Crash, false)
2006 }
2007 };
2008 (module_id, routes, reason, drained, terminal)
2009 });
2010 let registrations = self.deregister_connection(connection_id, end_reason);
2011 let cleanup = if crash_closed.is_some() {
2012 self.forwarding.cleanup_connection_counted(connection_id)
2013 } else {
2014 self.forwarding
2020 .cleanup_connection(connection_id)
2021 .map(|released| crate::forwarding::ConnectionCleanup {
2022 released,
2023 abandoned_relays: 0,
2024 })
2025 };
2026 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2031 let abandoned = cleanup
2032 .as_ref()
2033 .map_or(0, |cleanup| cleanup.abandoned_relays);
2034 send_route_control_pushes(
2035 &self.forwarding,
2036 routes,
2037 ClientControlPush::RouteClosed {
2038 module_id,
2039 channels: Vec::new(),
2040 reason,
2041 drained,
2042 abandoned,
2043 excluded_subscriptions: 0,
2044 terminal: Some(terminal),
2045 },
2046 );
2047 }
2048 if let Ok(cleanup) = cleanup {
2049 self.emit_route_goodbyes(cleanup.released);
2050 }
2051 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2055 crate::supervise::notify_registration_release();
2056 self.capability_evaluator.wake_deadline_loop();
2057 self.refresh_capability_requirements();
2058 }
2059 self.supervisor.remove_spawn_subscribers(connection_id);
2060 self.activity_streams
2061 .lock()
2062 .unwrap_or_else(|p| p.into_inner())
2063 .retain(|(id, _), task| {
2064 if *id == connection_id {
2065 task.abort();
2066 false
2067 } else {
2068 true
2069 }
2070 });
2071 self.hello_launch_nonces
2074 .lock()
2075 .unwrap_or_else(|poisoned| poisoned.into_inner())
2076 .forget(connection_id);
2077 self.scopes
2078 .write()
2079 .unwrap_or_else(|poisoned| poisoned.into_inner())
2080 .release_connection(connection_id);
2081 registrations
2082 }
2083
2084 pub(crate) fn handle_route_goodbye(
2085 &self,
2086 connection_id: ConnectionId,
2087 route_channel: u16,
2088 route_epoch: u32,
2089 ) -> Result<bool, RouterError> {
2090 debug!(
2091 connection_id = connection_id.get(),
2092 route_channel, route_epoch, "handling route GOODBYE"
2093 );
2094 let RouteRelease::Removed(released_route) = self
2095 .forwarding
2096 .release_client_route(connection_id, route_channel, route_epoch)
2097 .map_err(RouterError::Forwarding)?
2098 else {
2099 return Ok(false);
2100 };
2101 self.emit_route_goodbyes(vec![released_route]);
2102 Ok(true)
2103 }
2104
2105 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2106 for released in released_routes {
2107 let frame = match Frame::build_with_version(
2108 released.negotiated_ver,
2109 FrameType::Goodbye,
2110 control_flags(),
2111 released.channel,
2112 released.epoch,
2113 0,
2114 Vec::new(),
2115 ) {
2116 Ok(frame) => frame,
2117 Err(err) => {
2118 warn!(
2119 route_channel = released.channel,
2120 error = %err,
2121 "failed to build route GOODBYE frame"
2122 );
2123 continue;
2124 }
2125 };
2126 if !released.close_on_delivery_failure() {
2127 crate::forwarding::send_module_route_goodbye(
2128 &self.counters,
2129 &released.sink,
2130 frame,
2131 released.module_id.as_deref(),
2132 "client route released",
2133 );
2134 continue;
2135 }
2136 if let Err(err) = released.sink.try_send(frame) {
2137 warn!(
2138 target_connection_id = released.connection_id.get(),
2139 route_channel = released.channel,
2140 error = %err,
2141 "route GOODBYE was not delivered to client; closing target connection"
2142 );
2143 if self
2144 .forwarding
2145 .escalate_client_delivery_failure(
2146 released.connection_id,
2147 released.channel,
2148 released.epoch,
2149 CloseReason::new(
2150 "route_goodbye_delivery_failed",
2151 format!(
2152 "failed to enqueue route GOODBYE for channel {}: {err}",
2153 released.channel
2154 ),
2155 ),
2156 crate::forwarding::UndeliveredFrame {
2157 module_id: released.module_id.as_deref(),
2158 sink: &released.sink,
2159 },
2160 )
2161 .unwrap_or(false)
2162 {
2163 self.counters.increment_goodbye_relay_client_failed();
2164 }
2165 }
2166 }
2167 }
2168
2169 fn send_abandoned_route_bind_goodbye(
2181 &self,
2182 module_sink: &crate::FrameSink,
2183 negotiated_ver: u8,
2184 module_channel: u16,
2185 module_epoch: u32,
2186 ) {
2187 let frame = match Frame::build_with_version(
2188 negotiated_ver,
2189 FrameType::Goodbye,
2190 control_flags(),
2191 module_channel,
2192 module_epoch,
2193 0,
2194 Vec::new(),
2195 ) {
2196 Ok(frame) => frame,
2197 Err(err) => {
2198 warn!(
2199 route_channel = module_channel,
2200 error = %err,
2201 "failed to build GOODBYE for abandoned route.bind"
2202 );
2203 return;
2204 }
2205 };
2206 crate::forwarding::send_module_route_goodbye(
2207 &self.counters,
2208 module_sink,
2209 frame,
2210 None,
2211 "abandoned route.bind",
2212 );
2213 }
2214
2215 fn handle_hello(
2216 &self,
2217 connection_id: ConnectionId,
2218 sink: Option<crate::FrameSink>,
2219 frame: Frame,
2220 ) -> Result<Vec<Frame>, RouterError> {
2221 debug!(
2222 connection_id = connection_id.get(),
2223 corr = frame.header.corr,
2224 "handling HELLO"
2225 );
2226 if self
2230 .registry
2231 .get_module_by_connection(connection_id)
2232 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2233 .is_some()
2234 {
2235 return Ok(vec![control_error_frame(
2236 &frame,
2237 "invalid_hello",
2238 "connection is already registered as a module",
2239 )?]);
2240 }
2241 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2242 Ok(value) => value,
2243 Err(err) => {
2244 return Ok(vec![control_error_frame(
2245 &frame,
2246 "invalid_hello",
2247 format!("malformed HELLO body: {err}"),
2248 )?])
2249 }
2250 };
2251 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2252 return Ok(vec![control_error_frame(
2253 &frame,
2254 "invalid_capability_grammar",
2255 err.to_string(),
2256 )?]);
2257 }
2258 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2259 return Ok(vec![control_error_frame(
2260 &frame,
2261 "invalid_manifest",
2262 err.to_string(),
2263 )?]);
2264 }
2265 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2266 return Ok(vec![control_error_frame(
2267 &frame,
2268 "invalid_event_declaration",
2269 err.to_string(),
2270 )?]);
2271 }
2272 if let Some(provenance) = hello_value
2273 .get("manifest")
2274 .and_then(|manifest| manifest.get("provenance"))
2275 {
2276 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2277 return Ok(vec![control_error_frame(
2278 &frame,
2279 "invalid_manifest",
2280 format!("malformed manifest provenance: {err}"),
2281 )?]);
2282 }
2283 }
2284 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2285 Ok(hello) => hello,
2286 Err(err) => {
2287 return Ok(vec![control_error_frame(
2288 &frame,
2289 "invalid_hello",
2290 format!("malformed HELLO body: {err}"),
2291 )?])
2292 }
2293 };
2294
2295 if hello.protocol_ver != hello.manifest.protocol_ver {
2296 return Ok(vec![control_error_frame(
2297 &frame,
2298 "invalid_manifest",
2299 format!(
2300 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2301 hello.protocol_ver, hello.manifest.protocol_ver
2302 ),
2303 )?]);
2304 }
2305
2306 if hello.manifest.module_id.trim().is_empty() {
2307 return Ok(vec![control_error_frame(
2308 &frame,
2309 "invalid_manifest",
2310 "manifest module_id must not be empty",
2311 )?]);
2312 }
2313
2314 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2315 Ok(negotiated_ver) => negotiated_ver,
2316 Err(message) => {
2317 return Ok(vec![control_error_frame(
2318 &frame,
2319 "version_unsupported",
2320 message,
2321 )?])
2322 }
2323 };
2324
2325 let swap_admission = self
2333 .supervisor
2334 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2335 if swap_admission == SwapHelloAdmission::Refused {
2336 warn!(
2337 module_id = %hello.manifest.module_id,
2338 connection_id = connection_id.get(),
2339 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2340 );
2341 return Ok(vec![control_error_frame(
2342 &frame,
2343 "swap_token_invalid",
2344 format!(
2345 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2346 hello.manifest.module_id
2347 ),
2348 )?]);
2349 }
2350 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2351
2352 if let Some(rejection) = (!swap_candidate)
2360 .then(|| {
2361 self.supervisor.reserved_hello_rejection(
2362 &hello.manifest.module_id,
2363 hello.launch_nonce.as_deref(),
2364 )
2365 })
2366 .flatten()
2367 {
2368 let message = match rejection {
2369 ReservedHelloRejection::Exact { module_id } => format!(
2370 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2371 ),
2372 ReservedHelloRejection::Prefix {
2373 prefix,
2374 owner_module_id,
2375 } => format!(
2376 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2377 hello.manifest.module_id
2378 ),
2379 };
2380 return Ok(vec![control_error_frame(
2381 &frame,
2382 "reserved_module",
2383 message,
2384 )?]);
2385 }
2386
2387 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2388 &hello.manifest.module_id,
2389 hello.manifest.capabilities.as_ref(),
2390 );
2391 if let Some(refusal) = reserved_capability_refusals.first() {
2392 let capability = refusal.capability.clone();
2393 let bound_module = refusal.claimants[0].clone();
2394 log_duplicate_claim_events(reserved_capability_refusals);
2395 return Ok(vec![control_error_frame(
2396 &frame,
2397 "reserved_capability",
2398 format!(
2399 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2400 capability, bound_module, hello.manifest.module_id
2401 ),
2402 )?]);
2403 }
2404
2405 if self
2408 .forwarding
2409 .connection_has_client_routes(connection_id)
2410 .map_err(RouterError::Forwarding)?
2411 {
2412 return Ok(vec![control_error_frame(
2413 &frame,
2414 "invalid_hello",
2415 "connection has open client routes and cannot also register as a module",
2416 )?]);
2417 }
2418
2419 self.hello_launch_nonces
2424 .lock()
2425 .unwrap_or_else(|poisoned| poisoned.into_inner())
2426 .record(connection_id, hello.launch_nonce.as_deref());
2427 let control_ops = effective_module_control_ops(hello.control_ops);
2428 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2431 if swap_candidate {
2432 return self.register_swap_candidate(
2433 connection_id,
2434 sink,
2435 &frame,
2436 hello.manifest,
2437 negotiated_ver,
2438 control_ops,
2439 hello_ack,
2440 );
2441 }
2442 let registration = match self.registry.register_unpublished_with_control_ops(
2449 hello.manifest,
2450 negotiated_ver,
2451 connection_id,
2452 control_ops,
2453 ) {
2454 Ok(registration) => registration,
2455 Err(RegistryError::DuplicateModuleId { module_id }) => {
2456 return Ok(vec![control_error_frame(
2457 &frame,
2458 "duplicate_module_id",
2459 format!(
2460 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2461 ),
2462 )?])
2463 }
2464 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2465 return Ok(vec![control_error_frame(
2466 &frame,
2467 "invalid_module_id",
2468 err.to_string(),
2469 )?])
2470 }
2471 Err(err) => {
2472 return Ok(vec![control_error_frame(
2473 &frame,
2474 "registry_error",
2475 err.to_string(),
2476 )?])
2477 }
2478 };
2479 #[cfg(test)]
2480 self.hold_between_registry_and_forwarding();
2481
2482 let reply = if let Some(sink) = sink {
2483 let concurrency = manifest_concurrency(®istration.manifest);
2499 if let Err(err) = self.forwarding.register_module_connection_acked(
2500 connection_id,
2501 registration.manifest.module_id.clone(),
2502 negotiated_ver,
2503 concurrency,
2504 sink,
2505 hello_ack,
2506 ) {
2507 if matches!(
2512 self.deregister_connection(
2513 connection_id,
2514 RegistrationEndReason::RegistrationFailed,
2515 ),
2516 Ok(r) if !r.is_empty()
2517 ) {
2518 crate::supervise::notify_registration_release();
2519 }
2520 return Ok(vec![control_error_frame(
2521 &frame,
2522 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2523 "invalid_hello"
2524 } else {
2525 forwarding_error_code(&err)
2526 },
2527 err.to_string(),
2528 )?]);
2529 }
2530 Vec::new()
2531 } else {
2532 vec![hello_ack]
2535 };
2536
2537 if !self
2542 .registry
2543 .publish_registration(connection_id)
2544 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2545 {
2546 debug!(
2547 module_id = %registration.manifest.module_id,
2548 connection_id = connection_id.get(),
2549 "registration was removed before it could be published"
2550 );
2551 }
2552
2553 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2562 info!(
2563 module_id = %registration.manifest.module_id,
2564 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2565 );
2566 }
2567
2568 self.apply_registration_capabilities(®istration);
2569
2570 self.supervisor
2571 .record_activity_registration(®istration.manifest.module_id);
2572
2573 info!(
2574 module_id = %registration.manifest.module_id,
2575 module_version = %registration.manifest.module_version,
2576 negotiated_ver,
2577 routable_provider = manifest_provides_routable_role(®istration.manifest),
2578 connection_id = connection_id.get(),
2579 "module registered"
2580 );
2581
2582 Ok(reply)
2583 }
2584
2585 #[allow(clippy::too_many_arguments)]
2602 fn register_swap_candidate(
2603 &self,
2604 connection_id: ConnectionId,
2605 sink: Option<crate::FrameSink>,
2606 frame: &Frame,
2607 manifest: ModuleManifest,
2608 negotiated_ver: u8,
2609 control_ops: Vec<String>,
2610 hello_ack: Frame,
2611 ) -> Result<Vec<Frame>, RouterError> {
2612 let module_id = manifest.module_id.clone();
2613 let registration = match self.registry.register_unpublished_candidate_with_control_ops(
2614 manifest,
2615 negotiated_ver,
2616 connection_id,
2617 control_ops,
2618 ) {
2619 Ok(registration) => registration,
2620 Err(RegistryError::DuplicateModuleId { module_id }) => {
2621 return Ok(vec![control_error_frame(
2622 frame,
2623 "duplicate_module_id",
2624 format!(
2625 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2626 ),
2627 )?])
2628 }
2629 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2630 return Ok(vec![control_error_frame(
2631 frame,
2632 "invalid_module_id",
2633 err.to_string(),
2634 )?])
2635 }
2636 Err(err) => {
2637 return Ok(vec![control_error_frame(
2638 frame,
2639 "registry_error",
2640 err.to_string(),
2641 )?])
2642 }
2643 };
2644 #[cfg(test)]
2645 self.hold_between_registry_and_forwarding();
2646 let reply = if let Some(sink) = sink {
2647 let concurrency = manifest_concurrency(®istration.manifest);
2651 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2652 connection_id,
2653 module_id.clone(),
2654 negotiated_ver,
2655 concurrency,
2656 sink,
2657 hello_ack,
2658 ) {
2659 if matches!(
2660 self.deregister_connection(
2661 connection_id,
2662 RegistrationEndReason::RegistrationFailed,
2663 ),
2664 Ok(r) if !r.is_empty()
2665 ) {
2666 crate::supervise::notify_registration_release();
2667 }
2668 return Ok(vec![control_error_frame(
2669 frame,
2670 forwarding_error_code(&err),
2671 err.to_string(),
2672 )?]);
2673 }
2674 Vec::new()
2675 } else {
2676 vec![hello_ack]
2677 };
2678 if !self
2679 .registry
2680 .publish_registration(connection_id)
2681 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2682 {
2683 debug!(
2684 module_id = %module_id,
2685 connection_id = connection_id.get(),
2686 "swap candidate registration was removed before it could be published"
2687 );
2688 }
2689 self.supervisor.mark_swap_candidate_admitted(&module_id);
2690 info!(
2691 module_id = %module_id,
2692 module_version = %registration.manifest.module_version,
2693 negotiated_ver,
2694 ready = registration.ready,
2695 connection_id = connection_id.get(),
2696 "swap candidate registered; not routable until cutover"
2697 );
2698 Ok(reply)
2699 }
2700
2701 fn build_hello_ack(
2702 &self,
2703 frame: &Frame,
2704 negotiated_ver: u8,
2705 module_id: &str,
2706 ) -> Result<Frame, RouterError> {
2707 let ack = ModuleHelloAckBody {
2708 negotiated_ver,
2709 subc_ops: module_subc_ops(),
2710 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2711 storage: self
2712 .storage_config
2713 .as_ref()
2714 .map(|cfg| cfg.descriptor_for(module_id)),
2715 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2716 };
2717 let body = serde_json::to_vec(&ack).map_err(|err| {
2718 RouterError::backend(
2719 0,
2720 frame.header.corr,
2721 format!("failed to encode HELLO_ACK: {err}"),
2722 )
2723 })?;
2724
2725 Frame::build_with_version(
2726 negotiated_ver,
2727 FrameType::HelloAck,
2728 control_flags(),
2729 0,
2730 0,
2731 frame.header.corr,
2732 body,
2733 )
2734 .map_err(RouterError::FrameBuild)
2735 }
2736
2737 async fn handle_client_control_request(
2738 &self,
2739 ctx: &RouteCtx,
2740 frame: Frame,
2741 request: ClientControlRequest,
2742 route_open_timings: &mut Option<RouteOpenTimings>,
2743 ) -> Result<Vec<Frame>, RouterError> {
2744 if let Some(operation) = privileged_operation(&request) {
2745 let caller = self.resolve_caller(ctx.connection_id).await;
2746 let op = client_control_request_op(&request);
2747 let text = prompt_text(operation, &caller);
2748 match self.privileged_operator_prompt {
2749 PrivilegedOperatorPrompt::Log => {
2750 info!(target: "subc_daemon::privileged_prompt", operation = op,
2751 target = ?operation, caller = %caller.description(), summary = %text,
2752 "privileged request would prompt the operator");
2753 }
2754 PrivilegedOperatorPrompt::Enforce => {
2755 let key =
2756 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2757 let outcome = self
2758 .forwarding
2759 .operator_confirms()
2760 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2761 .await;
2762 if outcome.code != "confirmed" {
2763 return Ok(vec![outcome.frame(&frame)]);
2764 }
2765 }
2766 }
2767 }
2768 match request {
2769 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2770 ClientControlRequest::CatalogList { module_id } => {
2771 self.handle_catalog_list(frame, module_id)
2772 }
2773 ClientControlRequest::RouteOpen {
2774 target,
2775 identity,
2776 consumer_identity,
2777 consumer_capabilities,
2778 role_versions,
2779 admission_facts,
2780 scope,
2781 } => {
2782 self.handle_route_open(
2783 ctx,
2784 frame,
2785 RouteOpenRequest {
2786 target,
2787 identity,
2788 consumer_identity,
2789 consumer_capabilities,
2790 role_versions,
2791 admission_facts,
2792 scope,
2793 },
2794 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2795 )
2796 .await
2797 }
2798 ClientControlRequest::RoutePoll {
2799 route_channel,
2800 route_epoch,
2801 kind,
2802 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2803 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2804 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2805 self.handle_supervisor_spawn_snapshot(frame)
2806 }
2807 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2808 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2809 }
2810 ClientControlRequest::SupervisorRestart {
2811 module_id,
2812 drain_timeout_ms,
2813 } => {
2814 self.log_supervisor_request_received(
2815 ctx,
2816 frame.header.corr,
2817 ops::SUPERVISOR_RESTART,
2818 Some(&module_id),
2819 None,
2820 )?;
2821 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2822 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2823 .await
2824 }
2825 ClientControlRequest::SupervisorSwap {
2826 module_id,
2827 ready_timeout_ms,
2828 } => {
2829 self.log_supervisor_request_received(
2830 ctx,
2831 frame.header.corr,
2832 ops::SUPERVISOR_SWAP,
2833 Some(&module_id),
2834 None,
2835 )?;
2836 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2837 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
2838 .await
2839 }
2840 ClientControlRequest::SupervisorReload { module_id } => {
2841 self.log_supervisor_request_received(
2842 ctx,
2843 frame.header.corr,
2844 ops::SUPERVISOR_RELOAD,
2845 Some(&module_id),
2846 None,
2847 )?;
2848 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2849 self.handle_supervisor_reload(frame, module_id, requester)
2850 .await
2851 }
2852 ClientControlRequest::SupervisorRescan { preview } => {
2853 if !preview {
2854 self.log_supervisor_request_received(
2855 ctx,
2856 frame.header.corr,
2857 ops::SUPERVISOR_RESCAN,
2858 None,
2859 None,
2860 )?;
2861 }
2862 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2863 self.handle_supervisor_rescan(frame, preview, requester)
2864 .await
2865 }
2866 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
2867 self.log_supervisor_request_received(
2868 ctx,
2869 frame.header.corr,
2870 ops::SUPERVISOR_RELEASE_RESERVED,
2871 Some(&module_id),
2872 None,
2873 )?;
2874 self.handle_supervisor_release_reserved(frame, module_id)
2875 .await
2876 }
2877 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
2878 self.log_supervisor_request_received(
2879 ctx,
2880 frame.header.corr,
2881 ops::SUPERVISOR_SET_ENABLED,
2882 Some(&module_id),
2883 Some(enabled),
2884 )?;
2885 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2886 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
2887 .await
2888 }
2889 ClientControlRequest::SupervisorHealthProbe { module_id } => {
2890 self.handle_supervisor_health_probe(frame, module_id).await
2891 }
2892 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
2893 ClientControlRequest::SupervisorRoutes { module_id } => {
2894 self.handle_supervisor_routes(frame, module_id)
2895 }
2896 ClientControlRequest::SupervisorProvenance { module_id } => {
2897 self.handle_supervisor_provenance(frame, module_id).await
2898 }
2899 ClientControlRequest::SupervisorStderrTail {
2900 module_id,
2901 max_lines,
2902 max_bytes,
2903 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
2904 ClientControlRequest::SupervisorTerminals { module_id } => {
2905 self.handle_supervisor_terminals(frame, module_id).await
2906 }
2907 }
2908 }
2909
2910 fn log_supervisor_request_received(
2911 &self,
2912 ctx: &RouteCtx,
2913 corr: u64,
2914 op: &'static str,
2915 module_id: Option<&str>,
2916 enabled: Option<bool>,
2917 ) -> Result<(), RouterError> {
2918 let caller = self.supervisor_requester(ctx, corr)?;
2919 let caller = principal_label(&caller);
2920 info!(
2923 target: "control",
2924 op,
2925 module_id,
2926 enabled,
2927 connection_id = ctx.connection_id.get(),
2928 caller = %caller,
2929 "supervisor request received"
2930 );
2931 Ok(())
2932 }
2933
2934 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
2935 Ok(self
2936 .registry
2937 .get_module_by_connection(ctx.connection_id)
2938 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
2939 .map(|registration| Principal::Reserved {
2940 module_id: registration.manifest.module_id,
2941 })
2942 .unwrap_or(Principal::Direct))
2943 }
2944
2945 fn handle_module_control_request(
2946 &self,
2947 connection_id: ConnectionId,
2948 frame: Frame,
2949 request: ModuleControlRequestFromModule,
2950 ) -> Result<Vec<Frame>, RouterError> {
2951 match request {
2952 ModuleControlRequestFromModule::CatalogUpdate {
2953 provides,
2954 capabilities,
2955 ready,
2956 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
2957 ModuleControlRequestFromModule::LiveRoots {} => {
2958 let registered = self
2959 .registry
2960 .get_module_by_connection(connection_id)
2961 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2962 let Some(registration) = registered else {
2963 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
2964 };
2965 let response = self
2966 .forwarding
2967 .live_roots(®istration.manifest.module_id)
2968 .map_err(RouterError::Forwarding)?;
2969 Ok(vec![control_response_body_frame(
2970 &frame,
2971 &response,
2972 "ModuleControlResponseToModule::LiveRoots",
2973 )?])
2974 }
2975 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
2976 self.handle_scope_sync(connection_id, frame, generation, scopes)
2977 }
2978 ModuleControlRequestFromModule::ScopeApply {
2979 generation,
2980 upsert,
2981 end,
2982 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
2983 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
2984 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
2985 }
2986 }
2987 }
2988
2989 fn handle_operator_confirm(
2990 &self,
2991 ctx: &RouteCtx,
2992 frame: Frame,
2993 ) -> Result<Vec<Frame>, RouterError> {
2994 use crate::operator_confirm::{audit, Outcome};
2995 let registration = self
2996 .registry
2997 .get_module_by_connection(ctx.connection_id)
2998 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
2999 let Some(registration) = registration else {
3000 let outcome = Outcome::refusal("not_registered");
3001 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
3002 return Ok(vec![outcome.frame(&frame)]);
3003 };
3004 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3005 Ok(request) => request,
3006 Err(_) => {
3007 let outcome = Outcome::refusal("invalid_control_body");
3008 audit("", "", "", outcome, Duration::ZERO, Duration::ZERO, false);
3009 return Ok(vec![outcome.frame(&frame)]);
3010 }
3011 };
3012 let module_id = registration.manifest.module_id;
3013 let nonce = self
3017 .hello_launch_nonces
3018 .lock()
3019 .unwrap_or_else(|p| p.into_inner())
3020 .nonce(ctx.connection_id)
3021 .map(str::to_owned);
3022 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3023 self.supervisor
3024 .spawned_consumer_authorized(&module_id, nonce)
3025 });
3026 let confirms = self.forwarding.operator_confirms();
3027 self.forwarding
3028 .with_operator_route(
3029 ctx.connection_id,
3030 request.route_channel,
3031 request.route_epoch,
3032 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3033 )
3034 .map_err(RouterError::Forwarding)
3035 }
3036
3037 fn handle_scope_sync(
3041 &self,
3042 connection_id: ConnectionId,
3043 frame: Frame,
3044 generation: u64,
3045 scopes: Vec<ScopeRecord>,
3046 ) -> Result<Vec<Frame>, RouterError> {
3047 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3048 }
3049
3050 fn handle_scope_change(
3051 &self,
3052 connection_id: ConnectionId,
3053 frame: Frame,
3054 generation: u64,
3055 scopes: Vec<ScopeRecord>,
3056 end: Option<Vec<ScopeEnd>>,
3057 ) -> Result<Vec<Frame>, RouterError> {
3058 let is_apply = end.is_some();
3059 let Some(registration) = self
3060 .registry
3061 .get_module_by_connection(connection_id)
3062 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3063 else {
3064 return Ok(vec![control_error_frame(
3065 &frame,
3066 "not_registered",
3067 "scope.sync and scope.apply require an active module registration owned by this connection",
3068 )?]);
3069 };
3070 let owner = registration.manifest.module_id;
3071 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3072 let is_current_launch = |connection: ConnectionId| {
3073 self.hello_launch_nonces
3074 .lock()
3075 .unwrap_or_else(|poisoned| poisoned.into_inner())
3076 .presented(connection, current_nonce.as_deref())
3077 };
3078 let mut table = self
3083 .scopes
3084 .write()
3085 .unwrap_or_else(|poisoned| poisoned.into_inner());
3086 let outcome = match end {
3087 Some(end) => table.apply(
3088 &owner,
3089 connection_id,
3090 is_current_launch,
3091 generation,
3092 scopes,
3093 end,
3094 ),
3095 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3096 };
3097 let drained = match &outcome {
3098 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3099 Err(_) => Vec::new(),
3100 };
3101 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3102 drop(table);
3103 if has_deadlines {
3104 self.scope_deadline_added.notify_one();
3105 }
3106 match outcome {
3107 Ok(applied) => {
3108 let counts = ScopeOutcomeCounts::of(&applied.results);
3109 info!(
3110 owner = %owner,
3111 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3112 generation,
3113 records = applied.results.len(),
3114 created = counts.created,
3115 replaced = counts.replaced,
3116 updated = counts.updated,
3117 unchanged = counts.unchanged,
3118 refused = counts.refused,
3119 ended = applied.ended.len(),
3120 tag_changes = applied.tag_changes.len(),
3121 routes_closed = drained.len(),
3122 "scope change accepted"
3123 );
3124 for refused in applied
3130 .results
3131 .iter()
3132 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3133 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3134 {
3135 warn!(
3136 owner = %owner,
3137 generation,
3138 scope_ref = %refused.scope_ref,
3139 scope_epoch = refused.scope_epoch,
3140 code = refused.code.as_deref().unwrap_or(""),
3141 "scope record refused"
3142 );
3143 }
3144 self.close_scope_drained_routes(drained);
3145 let response = if is_apply {
3146 ModuleControlResponseToModule::ScopeApply {
3147 generation,
3148 results: applied.results,
3149 end_results: applied.end_results,
3150 ended: applied.ended,
3151 }
3152 } else {
3153 ModuleControlResponseToModule::ScopeSync {
3154 generation,
3155 results: applied.results,
3156 ended: applied.ended,
3157 }
3158 };
3159 Ok(vec![control_response_body_frame(
3160 &frame,
3161 &response,
3162 "scope change response",
3163 )?])
3164 }
3165 Err(refusal) => {
3166 info!(
3167 owner = %owner,
3168 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3169 generation,
3170 code = refusal.code,
3171 "scope change refused"
3172 );
3173 Ok(vec![control_error_frame(
3174 &frame,
3175 refusal.code,
3176 refusal.message,
3177 )?])
3178 }
3179 }
3180 }
3181
3182 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3188 if drained.is_empty() {
3189 return;
3190 }
3191 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3192 BTreeMap::new();
3193 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3194 for route in drained {
3195 warn!(
3196 module_id = %route.module_id,
3197 reason = ?route.reason,
3198 client_connection_id = route.client.connection_id.get(),
3199 route_channel = route.client.channel,
3200 "closing route because its scope changed"
3201 );
3202 pushes
3203 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3204 .or_insert_with(|| (route.reason, Vec::new()))
3205 .1
3206 .push(EndpointRoute {
3207 goodbye_target: route.client.clone(),
3208 principal: Principal::Unverified,
3209 bound_at: Instant::now(),
3210 draining: false,
3211 drain_reason: None,
3212 });
3213 goodbyes.push(route.module);
3214 goodbyes.push(route.client);
3215 }
3216 for ((module_id, _), (reason, routes)) in pushes {
3217 send_route_control_pushes(
3218 &self.forwarding,
3219 routes,
3220 ClientControlPush::RouteClosed {
3221 module_id,
3222 channels: Vec::new(),
3223 reason,
3224 drained: false,
3225 abandoned: 0,
3226 excluded_subscriptions: 0,
3227 terminal: Some(false),
3228 },
3229 );
3230 }
3231 self.emit_route_goodbyes(goodbyes);
3232 }
3233
3234 fn handle_scope_describe(
3237 &self,
3238 connection_id: ConnectionId,
3239 frame: Frame,
3240 owner: Principal,
3241 scope_ref: String,
3242 ) -> Result<Vec<Frame>, RouterError> {
3243 let registered = self
3244 .registry
3245 .get_module_by_connection(connection_id)
3246 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3247 if registered.is_none() {
3248 return Ok(vec![control_error_frame(
3249 &frame,
3250 "not_registered",
3251 "scope.describe requires an active module registration owned by this connection",
3252 )?]);
3253 }
3254 let description = self
3255 .scopes
3256 .read()
3257 .unwrap_or_else(|poisoned| poisoned.into_inner())
3258 .describe(&owner, &scope_ref);
3259 let owner_configured = match &owner {
3260 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3265 _ => false,
3266 };
3267 let response = ModuleControlResponseToModule::ScopeDescribe {
3268 status: description.status,
3269 scope_epoch: description.scope_epoch,
3270 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3271 owner_synced: description.owner_synced,
3272 owner_configured,
3273 scope: description.stamp,
3274 };
3275 Ok(vec![control_response_body_frame(
3276 &frame,
3277 &response,
3278 "ModuleControlResponseToModule::ScopeDescribe",
3279 )?])
3280 }
3281
3282 fn handle_catalog_update(
3283 &self,
3284 connection_id: ConnectionId,
3285 frame: Frame,
3286 provides: Vec<ProviderRole>,
3287 capabilities: Option<CapabilityDeclarations>,
3288 ready: Option<bool>,
3289 ) -> Result<Vec<Frame>, RouterError> {
3290 self.refresh_capability_requirements();
3291 let Some(registration) = self
3292 .registry
3293 .get_module_by_connection(connection_id)
3294 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3295 else {
3296 return Ok(vec![control_error_frame(
3297 &frame,
3298 "not_registered",
3299 "catalog.update requires an active module registration owned by this connection",
3300 )?]);
3301 };
3302
3303 if let Some(message) =
3304 catalog_update_frozen_field_message(®istration.manifest, &provides)
3305 {
3306 return Ok(vec![control_error_frame(
3307 &frame,
3308 "catalog_update_frozen_field",
3309 message,
3310 )?]);
3311 }
3312
3313 let mut candidate = registration.manifest.clone();
3314 candidate.provides = provides.clone();
3315 candidate.capabilities = capabilities
3316 .clone()
3317 .or_else(|| registration.manifest.capabilities.clone());
3318 if let Err(err) = candidate.validate_capability_grammar() {
3319 return Ok(vec![control_error_frame(
3320 &frame,
3321 "invalid_capability_grammar",
3322 err.to_string(),
3323 )?]);
3324 }
3325
3326 let mut conflicts = self
3329 .capability_evaluator
3330 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3331 if let Some(conflict) = conflicts.first() {
3332 let message = format!(
3333 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3334 conflict.capability, conflict.claimants[0], candidate.module_id
3335 );
3336 for conflict in &mut conflicts {
3337 conflict.source = DuplicateClaimSource::CatalogUpdate;
3338 }
3339 log_duplicate_claim_events(conflicts);
3340 return Ok(vec![control_error_frame(
3341 &frame,
3342 "reserved_capability",
3343 message,
3344 )?]);
3345 }
3346
3347 let updated = self
3348 .registry
3349 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3350 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3351 if updated.is_none() {
3352 return Ok(vec![control_error_frame(
3353 &frame,
3354 "not_registered",
3355 "catalog.update requires an active module registration owned by this connection",
3356 )?]);
3357 }
3358 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3359 log_duplicate_claim_events(
3360 self.capability_evaluator
3361 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3362 );
3363 }
3364 if capability_census_trigger(
3365 registration.manifest.capabilities.as_ref(),
3366 updated
3367 .as_ref()
3368 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3369 ) {
3370 self.enforce_capability_denies();
3371 }
3372 self.refresh_capability_requirements();
3373
3374 let response = ModuleControlResponseToModule::CatalogUpdate {};
3375 control_response_body_frame(
3376 &frame,
3377 &response,
3378 "ModuleControlResponseToModule::CatalogUpdate",
3379 )
3380 .map(|frame| vec![frame])
3381 }
3382
3383 pub(crate) fn with_activity_ledger_config(
3384 mut self,
3385 run_dir: &Path,
3386 incarnation: String,
3387 config: crate::activity::ActivityConfig,
3388 ) -> Self {
3389 self.activity = Some(crate::activity::ActivityLedger::open_config(
3390 run_dir.join("activity.jsonl"),
3391 incarnation,
3392 config,
3393 ));
3394 self.supervisor.set_activity(self.activity.clone());
3395 self
3396 }
3397
3398 pub(crate) async fn configure_activity(
3399 &self,
3400 config: crate::activity::ActivityConfig,
3401 ) -> Result<(), &'static str> {
3402 match &self.activity {
3403 Some(ledger) => ledger.configure(config).await,
3404 None => Ok(()),
3405 }
3406 }
3407
3408 fn cancel_activity_subscription(
3409 &self,
3410 ctx: &RouteCtx,
3411 frame: &Frame,
3412 ) -> Result<bool, RouterError> {
3413 let task = self
3414 .activity_streams
3415 .lock()
3416 .unwrap_or_else(|p| p.into_inner())
3417 .remove(&(ctx.connection_id, frame.header.corr));
3418 let Some(task) = task else {
3419 return Ok(false);
3420 };
3421 task.abort();
3422 let end = Frame::build_with_version(
3423 response_version(frame),
3424 FrameType::StreamEnd,
3425 control_flags(),
3426 0,
3427 0,
3428 frame.header.corr,
3429 Vec::new(),
3430 )
3431 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3432 let sink = ctx.egress.clone();
3433 tokio::spawn(async move {
3434 let _ = sink.send(end).await;
3435 });
3436 Ok(true)
3437 }
3438
3439 async fn handle_activity_read(
3440 &self,
3441 ctx: &RouteCtx,
3442 frame: Frame,
3443 ) -> Result<Vec<Frame>, RouterError> {
3444 use subc_control::{
3445 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3446 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3447 };
3448 #[derive(Deserialize)]
3449 struct Identity {
3450 #[serde(default)]
3451 consumer_identity: Option<ConsumerIdentity>,
3452 }
3453 let principal = match self
3457 .forwarding
3458 .module_id_for_connection(ctx.connection_id)
3459 .map_err(RouterError::Forwarding)?
3460 {
3461 Some(module_id) => Ok(Principal::Reserved { module_id }),
3462 None => {
3463 let identity: Identity = match serde_json::from_slice(&frame.body) {
3464 Ok(identity) => identity,
3465 Err(_) => {
3466 return Ok(vec![control_error_frame(
3467 &frame,
3468 ACTIVITY_MALFORMED,
3469 "malformed activity read",
3470 )?])
3471 }
3472 };
3473 self.route_open_principal(&frame, identity.consumer_identity)?
3474 }
3475 };
3476 let admitted = matches!(principal, Ok(Principal::Direct))
3477 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3478 if !admitted {
3479 return Ok(vec![control_error_frame(
3480 &frame,
3481 ACTIVITY_READ_FORBIDDEN,
3482 "activity reads require an operator or trusted mirror principal",
3483 )?]);
3484 }
3485 let Some(ledger) = &self.activity else {
3486 return Ok(vec![control_error_frame(
3487 &frame,
3488 ACTIVITY_UNAVAILABLE,
3489 "activity ledger unavailable",
3490 )?]);
3491 };
3492 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3493 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3494 {
3495 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3496 Ok(request) => ledger.query(request).await,
3497 Err(_) => Err(ACTIVITY_MALFORMED),
3498 };
3499 return match result {
3500 Ok(reply) => Ok(vec![control_response_body_frame(
3501 &frame,
3502 &reply,
3503 ops::ACTIVITY_QUERY,
3504 )?]),
3505 Err(code) => Ok(vec![control_error_frame(
3506 &frame,
3507 code,
3508 "activity query refused",
3509 )?]),
3510 };
3511 }
3512 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3513 Ok(request) => request,
3514 Err(_) => {
3515 return Ok(vec![control_error_frame(
3516 &frame,
3517 ACTIVITY_MALFORMED,
3518 "malformed activity subscription",
3519 )?])
3520 }
3521 };
3522 let mut subscription = match ledger.subscribe(request.since).await {
3523 Ok(subscription) => subscription,
3524 Err(code) => {
3525 return Ok(vec![control_error_frame(
3526 &frame,
3527 code,
3528 "activity subscription refused",
3529 )?])
3530 }
3531 };
3532 let key = (ctx.connection_id, frame.header.corr);
3533 let sink = ctx.egress.clone();
3534 let mut streams = self
3535 .activity_streams
3536 .lock()
3537 .unwrap_or_else(|p| p.into_inner());
3538 streams.retain(|_, task| !task.is_finished());
3539 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3540 return Ok(vec![control_error_frame(
3541 &frame,
3542 "duplicate_subscription",
3543 "activity subscription correlation id already in use",
3544 )?]);
3545 }
3546 let task = tokio::spawn(async move {
3547 loop {
3548 let entry = if let Some(entry) = subscription.replay.pop_front() {
3549 Some(entry)
3550 } else {
3551 subscription.receiver.recv().await
3552 };
3553 let Some(entry) = entry else {
3554 break;
3555 };
3556 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3557 return;
3558 };
3559 let Ok(next) = Frame::build_with_version(
3560 response_version(&frame),
3561 FrameType::StreamData,
3562 control_flags(),
3563 0,
3564 0,
3565 frame.header.corr,
3566 body,
3567 ) else {
3568 return;
3569 };
3570 if sink.send(next).await.is_err() {
3571 return;
3572 }
3573 }
3574 if let Some(code) = subscription.terminal_code() {
3577 if let Ok(end) =
3578 control_error_frame(&frame, code, "activity subscriber fell behind")
3579 {
3580 let _ = sink.send(end).await;
3581 }
3582 }
3583 });
3584 streams.insert(key, task.abort_handle());
3585 Ok(Vec::new())
3586 }
3587
3588 async fn handle_activity_record(
3589 &self,
3590 ctx: &RouteCtx,
3591 frame: Frame,
3592 ) -> Result<Vec<Frame>, RouterError> {
3593 use subc_control::{
3594 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3595 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3596 };
3597 let result = async {
3598 let module = self
3599 .forwarding
3600 .module_id_for_connection(ctx.connection_id)
3601 .ok()
3602 .flatten()
3603 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3604 let request: ActivityRecordRequest =
3605 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3606 crate::activity::validate(&module, &request)?;
3607 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3608 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3609 let at_ms = std::time::SystemTime::now()
3610 .duration_since(std::time::UNIX_EPOCH)
3611 .unwrap_or_default()
3612 .as_millis() as u64;
3613 ledger.record(module, request, actor, at_ms).await
3614 }
3615 .await;
3616 match result {
3617 Ok(entry) => Ok(vec![control_response_body_frame(
3618 &frame,
3619 &ActivityRecordResponse { seq: entry.seq },
3620 "activity.record",
3621 )?]),
3622 Err(code) => Ok(vec![control_error_frame(
3623 &frame,
3624 code,
3625 "activity recording refused",
3626 )?]),
3627 }
3628 }
3629
3630 fn activity_actor(
3631 &self,
3632 connection: ConnectionId,
3633 module: &str,
3634 on: Option<&subc_control::ActivityOn>,
3635 ) -> Result<subc_control::ActivityActor, &'static str> {
3636 use crate::forwarding::{DataRoute, DataRouteState};
3637 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3638 let principal = |id: &str| Principal::Reserved {
3639 module_id: id.to_owned(),
3640 };
3641 let mut actor = ActivityActor {
3642 principal: principal(module),
3643 scope_owner: None,
3644 agent_id: None,
3645 run_id: None,
3646 flow_id: None,
3647 project_root: None,
3648 };
3649 let scope = match on {
3650 None => return Ok(actor),
3651 Some(ActivityOn::Scope(scope)) => {
3652 let selector = subc_protocol::scope::ScopeSelector::new(
3653 principal(module),
3654 scope.scope_ref.clone(),
3655 Some(scope.epoch),
3656 );
3657 self.scopes
3658 .read()
3659 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3660 .attribute(&principal(module), module, &selector, true)
3661 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3662 }
3663 Some(ActivityOn::Route(route)) => {
3664 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3665 .forwarding
3666 .lookup_data_route(connection, route.channel, route.epoch)
3667 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3668 else {
3669 return Err(ACTIVITY_ACTOR_UNKNOWN);
3670 };
3671 if binding.module_id != module {
3672 return Err(ACTIVITY_ACTOR_UNKNOWN);
3673 }
3674 actor.principal = binding.principal.clone();
3675 actor.project_root = binding
3676 .project_root
3677 .as_ref()
3678 .map(|root| root.as_path().to_path_buf());
3679 let Some(bound) = &binding.scope else {
3680 return Ok(actor);
3681 };
3682 let owner = principal(&bound.owner);
3683 let selector = subc_protocol::scope::ScopeSelector::new(
3684 owner.clone(),
3685 bound.scope_ref.clone(),
3686 Some(bound.tag.scope_epoch),
3687 );
3688 let admitted = self
3689 .scopes
3690 .read()
3691 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3692 .attribute(&owner, module, &selector, true)
3693 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
3694 if admitted.tag != bound.tag {
3695 return Err(ACTIVITY_ACTOR_UNKNOWN);
3696 }
3697 admitted
3698 }
3699 };
3700 actor.scope_owner = Some(scope.stamp.owner);
3701 actor.agent_id = scope.stamp.attributes.agent_id;
3702 actor.run_id = scope.stamp.attributes.run_id;
3703 actor.flow_id = scope.stamp.attributes.flow_id;
3704 Ok(actor)
3705 }
3706
3707 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
3708 self.refresh_capability_requirements();
3709 let mut counters = self.counters.snapshot();
3715 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
3716 obj.insert(
3717 "activity_unparseable_lines".into(),
3718 ledger.read_errors().into(),
3719 );
3720 obj.insert(
3721 "activity_lifecycle_append_failures".into(),
3722 ledger.lifecycle_failures().into(),
3723 );
3724 obj.insert(
3725 "activity_started_digest_skips".into(),
3726 ledger.lifecycle_digest_skips().into(),
3727 );
3728 }
3729 if let (Ok((connections_with_routes, max)), Some(obj)) = (
3730 self.forwarding.client_route_concentration(),
3731 counters.as_object_mut(),
3732 ) {
3733 obj.insert(
3734 "client_connections_with_routes".into(),
3735 connections_with_routes.into(),
3736 );
3737 obj.insert("max_routes_on_one_connection".into(), max.into());
3738 }
3739 if let (Some(open_breakers), Some(obj)) = (
3745 self.route_bind_breakers.open_snapshot(),
3746 counters.as_object_mut(),
3747 ) {
3748 obj.insert("route_bind_breakers_open".into(), open_breakers);
3749 }
3750 let response = ClientControlResponse::ServerDescribe {
3751 protocol_ver: PROTOCOL_VERSION,
3752 subc_ops: subc_ops(),
3753 capabilities: self.subc_capabilities.as_ref().to_vec(),
3754 connected_clients: self.connected_clients.count(),
3755 counters: Some(counters),
3756 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
3757 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
3758 capability_requirements: self.capability_requirement_statuses(),
3759 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3760 };
3761 Ok(vec![control_response_body_frame(
3762 &frame,
3763 &response,
3764 "ClientControlResponse::ServerDescribe",
3765 )?])
3766 }
3767
3768 fn handle_catalog_list(
3769 &self,
3770 frame: Frame,
3771 module_id: Option<String>,
3772 ) -> Result<Vec<Frame>, RouterError> {
3773 let (generation, modules) = self.registry.list_modules().map_err(|err| {
3774 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
3775 })?;
3776 let entries = modules
3777 .into_iter()
3778 .filter(|registration| {
3779 module_id
3780 .as_deref()
3781 .map(|wanted| registration.manifest.module_id == wanted)
3782 .unwrap_or(true)
3783 })
3784 .map(|registration| {
3785 let not_ready = self.not_ready_reason(®istration);
3786 let roles = registration.manifest.provides;
3787 CatalogEntry::new(
3788 registration.manifest.module_id,
3789 roles,
3790 registration.control_ops,
3791 )
3792 .with_ready(not_ready.is_none())
3793 .with_not_ready(not_ready)
3794 .with_module_version(Some(registration.manifest.module_version))
3795 .with_capabilities(registration.manifest.capabilities)
3796 .with_self_signals(registration.manifest.self_signals)
3797 .with_events(registration.manifest.events)
3798 })
3799 .collect();
3800 let response = ClientControlResponse::CatalogList {
3801 generation,
3802 modules: entries,
3803 subc_ops: subc_ops(),
3804 };
3805 Ok(vec![control_response_body_frame(
3806 &frame,
3807 &response,
3808 "ClientControlResponse::CatalogList",
3809 )?])
3810 }
3811
3812 fn route_open_principal(
3813 &self,
3814 frame: &Frame,
3815 consumer_identity: Option<ConsumerIdentity>,
3816 ) -> Result<Result<Principal, Frame>, RouterError> {
3817 let Some(consumer_identity) = consumer_identity else {
3818 return Ok(Ok(Principal::Direct));
3819 };
3820
3821 if self.supervisor.spawned_consumer_authorized(
3822 &consumer_identity.module_id,
3823 &consumer_identity.launch_nonce,
3824 ) {
3825 return Ok(Ok(Principal::Reserved {
3826 module_id: consumer_identity.module_id,
3827 }));
3828 }
3829
3830 Ok(Err(control_error_frame(
3831 frame,
3832 "bad_consumer_identity",
3833 format!(
3834 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
3835 consumer_identity.module_id
3836 ),
3837 )?))
3838 }
3839
3840 fn route_open_refusal_frame(
3847 &self,
3848 ctx: &RouteCtx,
3849 frame: &Frame,
3850 module_id: &str,
3851 reason: &'static str,
3852 code: &'static str,
3853 message: impl Into<String>,
3854 ) -> Result<Frame, RouterError> {
3855 self.observe_route_open_refusal(ctx, module_id, reason, code);
3856 control_error_frame(frame, code, message.into())
3857 }
3858
3859 fn route_open_breaker_refusal_frame(
3877 &self,
3878 ctx: &RouteCtx,
3879 frame: &Frame,
3880 module_id: &str,
3881 consecutive_timeouts: u32,
3882 retry_in: Duration,
3883 probe_in_flight: bool,
3884 ) -> Result<Frame, RouterError> {
3885 self.counters
3886 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
3887 debug!(
3888 target: "control",
3889 code = "module_timeout",
3890 module_id = ?module_id,
3891 connection_id = ctx.connection_id.get(),
3892 consecutive_timeouts,
3893 retry_in_ms = retry_in.as_millis() as u64,
3894 probe_in_flight,
3895 "route.open refused by open bind-relay breaker"
3896 );
3897 let detail = if probe_in_flight {
3904 "one test route is already being tried; retry once it settles".to_string()
3905 } else {
3906 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
3907 };
3908 control_error_frame(
3909 frame,
3910 "module_timeout",
3911 format!(
3912 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
3913 timed out in a row); {detail}; its established routes are unaffected"
3914 ),
3915 )
3916 }
3917
3918 fn observe_route_open_refusal(
3931 &self,
3932 ctx: &RouteCtx,
3933 module_id: &str,
3934 reason: &'static str,
3935 code: &'static str,
3936 ) {
3937 self.counters.increment_route_open_refused(code);
3938 info!(
3939 target: "control",
3940 code,
3941 reason,
3942 module_id = ?module_id,
3943 connection_id = ctx.connection_id.get(),
3944 "route.open refused"
3945 );
3946 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
3947 self.route_outages.record_not_serving(module_id, reason);
3948 }
3949 }
3950
3951 fn observe_route_open_accept(&self, ctx: &RouteCtx, module_id: &str, principal: &str) {
3996 self.route_outages.record_accepted(module_id);
3997 self.counters.increment_route_open_accepted(principal);
3998 info!(
3999 target: "control",
4000 principal,
4001 module_id,
4002 connection_id = ctx.connection_id.get(),
4003 "route.open accepted"
4004 );
4005 }
4006
4007 fn supervised_absent_route_open_refusal_frame(
4008 &self,
4009 ctx: &RouteCtx,
4010 frame: &Frame,
4011 module_id: &str,
4012 code: &'static str,
4013 status: &crate::supervise::ModuleStatus,
4014 ) -> Result<Frame, RouterError> {
4015 self.counters.increment_route_open_refused(code);
4016 info!(
4017 target: "control",
4018 code,
4019 reason = "supervised_not_registered",
4020 module_id = ?module_id,
4021 connection_id = ctx.connection_id.get(),
4022 state = %status.state,
4023 enabled = status.enabled,
4024 live = status.live,
4025 "route.open refused"
4026 );
4027 self.route_outages
4031 .record_not_serving(module_id, "supervised_not_registered");
4032 let message = if let Some(pid) = status.waiting_for_previous_pid {
4033 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4034 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4035 })
4036 } else {
4037 format!(
4038 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4039 status.state, status.enabled, status.live
4040 )
4041 };
4042 control_error_frame(frame, code, message)
4043 }
4044
4045 async fn handle_route_open(
4046 &self,
4047 ctx: &RouteCtx,
4048 frame: Frame,
4049 request: RouteOpenRequest,
4050 timings: &mut RouteOpenTimings,
4051 ) -> Result<Vec<Frame>, RouterError> {
4052 let RouteOpenRequest {
4053 target,
4054 mut identity,
4055 consumer_identity,
4056 consumer_capabilities,
4057 role_versions,
4058 admission_facts,
4059 scope,
4060 } = request;
4061 let target_module_id = target_module_id(&target).to_string();
4062 timings.module_id.clone_from(&target_module_id);
4063 if self
4064 .registry
4065 .get_module_by_connection(ctx.connection_id)
4066 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4067 .is_some()
4068 {
4069 return Ok(vec![control_error_frame(
4070 &frame,
4071 "invalid_request",
4072 "module connections cannot open client routes",
4073 )?]);
4074 }
4075 debug!(
4076 connection_id = ctx.connection_id.get(),
4077 corr = frame.header.corr,
4078 module_id = %target_module_id,
4079 "handling route.open"
4080 );
4081
4082 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4088 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4089 self.observe_route_open_refusal(
4090 ctx,
4091 &target_module_id,
4092 "invalid_role_versions",
4093 error_codes::INVALID_REQUEST,
4094 );
4095 return Ok(vec![control_error_body_frame(
4096 &frame,
4097 ErrorBody {
4098 code: error_codes::INVALID_REQUEST.to_string(),
4099 message: error.to_string(),
4100 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4101 },
4102 )?]);
4103 }
4104
4105 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4106 if status.waiting_for_previous_pid.is_some() {
4107 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4108 ctx,
4109 &frame,
4110 &target_module_id,
4111 "module_warming",
4112 &status,
4113 )?]);
4114 }
4115 }
4116
4117 let Some(registration) = self
4136 .registry
4137 .get_module(&target_module_id)
4138 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4139 else {
4140 if let Some((status, warming)) =
4141 self.supervisor_status(&target_module_id, frame.header.corr)?
4142 {
4143 if status.protocol == ModuleProtocol::None {
4150 return Ok(vec![self.route_open_refusal_frame(
4151 ctx,
4152 &frame,
4153 &target_module_id,
4154 "protocol_none",
4155 error_codes::MODULE_NO_PROTOCOL,
4156 format!(
4157 "module_id '{target_module_id}' is declared protocol: none; \
4158 it speaks no subc wire and serves no routes"
4159 ),
4160 )?]);
4161 }
4162 let code = if warming {
4163 "module_warming"
4164 } else {
4165 "target_unavailable"
4166 };
4167 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4168 ctx,
4169 &frame,
4170 &target_module_id,
4171 code,
4172 &status,
4173 )?]);
4174 }
4175 if let Some(removed_ago_ms) =
4176 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4177 {
4178 return Ok(vec![self.route_open_refusal_frame(
4179 ctx,
4180 &frame,
4181 &target_module_id,
4182 "removed",
4183 error_codes::MODULE_REMOVED,
4184 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4185 )?]);
4186 }
4187 return Ok(vec![self.route_open_refusal_frame(
4188 ctx,
4189 &frame,
4190 &target_module_id,
4191 "not_registered",
4192 error_codes::UNKNOWN_MODULE,
4193 format!("module_id '{target_module_id}' is not registered"),
4194 )?]);
4195 };
4196
4197 if !registration.ready {
4201 self.counters
4202 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4203 info!(
4204 target: "control",
4205 code = error_codes::MODULE_WARMING,
4206 module_id = ?target_module_id,
4207 connection_id = ctx.connection_id.get(),
4208 reason = "declared_not_ready",
4209 "route.open refused"
4210 );
4211 self.route_outages
4214 .record_not_serving(&target_module_id, "declared_not_ready");
4215 return Ok(vec![control_error_body_frame(
4216 &frame,
4217 ErrorBody {
4218 code: error_codes::MODULE_WARMING.to_string(),
4219 message: format!(
4220 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4221 ),
4222 detail: Some(serde_json::json!({
4223 "reason": "declared_not_ready"
4224 })),
4225 },
4226 )?]);
4227 }
4228
4229 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4248 self.counters
4249 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4250 info!(
4251 target: "control",
4252 code = error_codes::MODULE_WARMING,
4253 module_id = ?target_module_id,
4254 connection_id = ctx.connection_id.get(),
4255 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4256 capability = %capability,
4257 "route.open refused"
4258 );
4259 return Ok(vec![control_error_body_frame(
4260 &frame,
4261 ErrorBody {
4262 code: error_codes::MODULE_WARMING.to_string(),
4263 message: format!(
4264 "module_id '{target_module_id}' requires capability '{capability}', \
4265 which no registered module provides; retry"
4266 ),
4267 detail: Some(serde_json::json!({
4268 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4269 "capability": capability,
4270 })),
4271 },
4272 )?]);
4273 }
4274
4275 if !target_has_required_role(&target, ®istration.manifest.provides) {
4276 return Ok(vec![self.route_open_refusal_frame(
4277 ctx,
4278 &frame,
4279 &target_module_id,
4280 "role_not_provided",
4281 "target_unavailable",
4282 format!("module_id '{target_module_id}' does not provide the requested target"),
4283 )?]);
4284 }
4285
4286 if registration.state != ChannelState::Active {
4287 return Ok(vec![self.route_open_refusal_frame(
4288 ctx,
4289 &frame,
4290 &target_module_id,
4291 "registration_not_active",
4292 "target_unavailable",
4293 format!("module_id '{target_module_id}' is not active"),
4294 )?]);
4295 }
4296
4297 if self
4298 .forwarding
4299 .module_is_draining(&target_module_id)
4300 .map_err(RouterError::Forwarding)?
4301 {
4302 return Ok(vec![self.route_open_refusal_frame(
4303 ctx,
4304 &frame,
4305 &target_module_id,
4306 "reloading",
4307 "module_reloading",
4308 format!("module_id '{target_module_id}' is reloading"),
4309 )?]);
4310 }
4311
4312 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4313 process_liveness.process_live(&target_module_id) == Some(false)
4314 }) {
4315 if process_liveness.process_replacing(&target_module_id) {
4322 return Ok(vec![self.route_open_refusal_frame(
4323 ctx,
4324 &frame,
4325 &target_module_id,
4326 "reloading",
4327 "module_reloading",
4328 format!("module_id '{target_module_id}' is reloading"),
4329 )?]);
4330 }
4331 return Ok(vec![self.route_open_refusal_frame(
4332 ctx,
4333 &frame,
4334 &target_module_id,
4335 "supervisor_not_live",
4336 "target_unavailable",
4337 format!("module_id '{target_module_id}' is not live"),
4338 )?]);
4339 }
4340
4341 if !self
4342 .forwarding
4343 .has_live_module_connection(&target_module_id)
4344 .map_err(RouterError::Forwarding)?
4345 {
4346 return Ok(vec![self.route_open_refusal_frame(
4347 ctx,
4348 &frame,
4349 &target_module_id,
4350 "no_forwarding_connection",
4351 "target_unavailable",
4352 format!("module_id '{target_module_id}' has no live forwarding connection"),
4353 )?]);
4354 }
4355
4356 if let Some(error) =
4357 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4358 {
4359 self.observe_route_open_refusal(
4360 ctx,
4361 &target_module_id,
4362 "op_not_allowed",
4363 "op_not_allowed",
4364 );
4365 return Ok(vec![error]);
4366 }
4367
4368 let principal = match self.route_open_principal(&frame, consumer_identity)? {
4369 Ok(principal) => principal,
4370 Err(error) => {
4371 self.observe_route_open_refusal(
4372 ctx,
4373 &target_module_id,
4374 "bad_consumer_identity",
4375 "bad_consumer_identity",
4376 );
4377 return Ok(vec![error]);
4378 }
4379 };
4380 timings.principal = principal_label(&principal);
4381
4382 if let Principal::Reserved {
4386 module_id: opening_module_id,
4387 } = &principal
4388 {
4389 if let Some(opening_registration) = self
4390 .registry
4391 .get_module(opening_module_id)
4392 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4393 {
4394 if let Some(capability) =
4395 denied_capability(&opening_registration.manifest, ®istration.manifest)
4396 {
4397 warn!(
4398 opening_module_id,
4399 target_module_id,
4400 capability,
4401 "refusing route.open because an attested capability deny edge matches"
4402 );
4403 return Ok(vec![self.route_open_refusal_frame(
4404 ctx,
4405 &frame,
4406 &target_module_id,
4407 "capability_deny_edge",
4408 "capability_forbidden",
4409 format!(
4410 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4411 ),
4412 )?]);
4413 }
4414 }
4415 }
4416
4417 if admission_facts.is_some() {
4418 let carrier_matches = matches!(
4419 &principal,
4420 Principal::Reserved { module_id }
4421 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4422 );
4423 if !carrier_matches {
4424 return Ok(vec![self.route_open_refusal_frame(
4425 ctx,
4426 &frame,
4427 &target_module_id,
4428 "admission_facts_carrier_not_permitted",
4429 "admission_facts_not_permitted",
4430 "admission facts may only be carried by the configured reserved module",
4431 )?]);
4432 }
4433
4434 let target_allowed = self
4435 .admission_facts_targets
4436 .as_ref()
4437 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4438 if !target_allowed {
4439 return Ok(vec![self.route_open_refusal_frame(
4440 ctx,
4441 &frame,
4442 &target_module_id,
4443 "admission_facts_target_not_listed",
4444 "admission_facts_target_not_allowed",
4445 format!(
4446 "admission facts are not permitted for target module_id '{target_module_id}'"
4447 ),
4448 )?]);
4449 }
4450
4451 }
4455
4456 let (bound_scope, mut scope_stamp) = match &scope {
4462 None => (None, None),
4463 Some(selector) => {
4464 let owner_configured = match &selector.owner {
4465 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4471 _ => false,
4472 };
4473 let mut scopes = self
4474 .scopes
4475 .write()
4476 .unwrap_or_else(|poisoned| poisoned.into_inner());
4477 let admitted = scopes
4478 .admit(&principal, &target_module_id, selector, owner_configured)
4479 .and_then(|admission| {
4480 crate::scopes::check_target_flow_support(
4481 &admission.stamp,
4482 &target_module_id,
4483 registration.manifest.capabilities.as_ref(),
4484 )?;
4485 crate::scopes::check_target_agent_run_support(
4486 &admission.stamp,
4487 &target_module_id,
4488 registration.manifest.capabilities.as_ref(),
4489 )?;
4490 let bound = scopes.reserve_bind(
4491 &admission,
4492 &principal,
4493 ctx.connection_id,
4494 selector.hold_presence,
4495 );
4496 Ok((admission, bound))
4497 });
4498 match admitted {
4499 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
4500 Err(refusal) => {
4501 return Ok(vec![self.route_open_refusal_frame(
4502 ctx,
4503 &frame,
4504 &target_module_id,
4505 refusal.code,
4506 refusal.code,
4507 refusal.message,
4508 )?]);
4509 }
4510 }
4511 }
4512 };
4513
4514 timings.enter(RouteOpenPhase::ProjectRoot);
4536 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
4537 Ok(project_root) => project_root,
4538 Err(err) => {
4539 return Ok(vec![control_error_frame(
4540 &frame,
4541 "invalid_project_root",
4542 err.to_string(),
4543 )?])
4544 }
4545 };
4546 timings.enter(RouteOpenPhase::Admission);
4547 identity.project_root = project_root.as_path().to_path_buf();
4548
4549 let _concurrency_guard = match self
4560 .route_bind_concurrency
4561 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
4562 {
4563 Ok(guard) => guard,
4564 Err(in_flight) => {
4565 return Ok(vec![self.route_open_target_capacity_refusal(
4566 ctx,
4567 &frame,
4568 &target_module_id,
4569 in_flight,
4570 )?]);
4571 }
4572 };
4573
4574 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
4577 RouteBindAdmission::Admitted { guard, probe } => {
4578 if probe {
4579 info!(
4580 module_id = %target_module_id,
4581 connection_id = ctx.connection_id.get(),
4582 "route.bind breaker half-open: admitting one probe"
4583 );
4584 }
4585 guard
4586 }
4587 RouteBindAdmission::Refused {
4588 consecutive_timeouts,
4589 retry_in,
4590 probe_in_flight,
4591 } => {
4592 return Ok(vec![self.route_open_breaker_refusal_frame(
4593 ctx,
4594 &frame,
4595 &target_module_id,
4596 consecutive_timeouts,
4597 retry_in,
4598 probe_in_flight,
4599 )?]);
4600 }
4601 };
4602
4603 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
4608 let relay_deadline = Instant::now() + route_bind_relay_timeout;
4609 timings.enter(RouteOpenPhase::EgressReservation);
4610 let pending = match self
4611 .forwarding
4612 .begin_route_bind_relay_for(
4613 ctx.connection_id,
4614 ctx.egress.clone(),
4615 response_version(&frame),
4616 frame.header.corr,
4617 &target_module_id,
4618 principal.clone(),
4619 bound_scope,
4620 Some(project_root),
4621 relay_deadline,
4622 )
4623 .await
4624 {
4625 Ok(pending) => pending,
4626 Err(err) => {
4627 return Ok(vec![self.route_open_refusal_frame(
4628 ctx,
4629 &frame,
4630 &target_module_id,
4631 "relay_reservation_failed",
4632 forwarding_error_code(&err),
4633 err.to_string(),
4634 )?])
4635 }
4636 };
4637 timings.enter(RouteOpenPhase::Admission);
4638 let crate::forwarding::PendingRouteBindRelay {
4639 endpoint,
4640 module_sink,
4641 negotiated_ver,
4642 client_channel,
4643 client_epoch,
4644 module_channel,
4645 module_epoch,
4646 corr: relay_corr,
4647 receiver,
4648 } = pending;
4649 let mut reservation =
4650 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
4651
4652 let relay_registration = self
4657 .registry
4658 .get_module_by_connection(endpoint.connection_id)
4659 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4660 let relay_capabilities = relay_registration
4661 .as_ref()
4662 .and_then(|registration| registration.manifest.capabilities.as_ref());
4663 if let Some(selector) = &scope {
4664 if let Err(refusal) = self
4667 .scopes
4668 .read()
4669 .unwrap_or_else(|poisoned| poisoned.into_inner())
4670 .check_bind_proof(&principal, selector)
4671 {
4672 reservation.release_and_disarm();
4673 return Ok(vec![self.route_open_refusal_frame(
4674 ctx,
4675 &frame,
4676 &target_module_id,
4677 refusal.code,
4678 refusal.code,
4679 refusal.message,
4680 )?]);
4681 }
4682 }
4683 if let Some(stamp) = scope_stamp
4684 .as_ref()
4685 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
4686 {
4687 if let Err(refusal) = crate::scopes::check_target_flow_support(
4688 stamp,
4689 &target_module_id,
4690 relay_capabilities,
4691 )
4692 .and_then(|()| {
4693 crate::scopes::check_target_agent_run_support(
4694 stamp,
4695 &target_module_id,
4696 relay_capabilities,
4697 )
4698 }) {
4699 reservation.release_and_disarm();
4700 return Ok(vec![self.route_open_refusal_frame(
4701 ctx,
4702 &frame,
4703 &target_module_id,
4704 refusal.code,
4705 refusal.code,
4706 refusal.message,
4707 )?]);
4708 }
4709 }
4710
4711 let call_mode = if relay_capabilities
4714 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
4715 {
4716 Some(derive_call_mode(scope_stamp.as_ref()))
4717 } else {
4718 if let Some(stamp) = scope_stamp.as_mut() {
4719 stamp.expires_at_ms = None;
4720 }
4721 None
4722 };
4723
4724 debug!(
4725 connection_id = ctx.connection_id.get(),
4726 client_channel,
4727 client_epoch,
4728 module_channel,
4729 module_epoch,
4730 "reserved route handle pair"
4731 );
4732 let principal_label = principal_label(&principal);
4735 let relay = ModuleControlRequest::RouteBind {
4736 route_channel: module_channel,
4737 epoch: module_epoch,
4738 target,
4739 identity,
4740 principal: Some(principal),
4741 consumer_capabilities,
4742 role_versions,
4743 admission_facts,
4744 scope: scope_stamp,
4745 call_mode,
4746 };
4747 let relay_body = serde_json::to_vec(&relay).map_err(|err| {
4748 RouterError::backend(
4749 0,
4750 frame.header.corr,
4751 format!("failed to encode route.bind request: {err}"),
4752 )
4753 })?;
4754 let relay_frame = Frame::build_with_version(
4755 negotiated_ver,
4756 FrameType::Request,
4757 control_flags(),
4758 0,
4759 0,
4760 relay_corr,
4761 relay_body,
4762 )
4763 .map_err(RouterError::FrameBuild)?;
4764
4765 timings.enter(RouteOpenPhase::BindSend);
4766 let sent = module_sink.send(relay_frame).await;
4767 timings.enter(RouteOpenPhase::Admission);
4768 if let Err(err) = sent {
4769 reservation.release_and_disarm();
4770 return Ok(vec![self.route_open_refusal_frame(
4771 ctx,
4772 &frame,
4773 &target_module_id,
4774 "relay_send_failed",
4775 "target_unavailable",
4776 err.to_string(),
4777 )?]);
4778 }
4779
4780 #[cfg(test)]
4781 if let Some(hold) = &self.route_bind_relay_sent_hold {
4782 hold.wait().await;
4786 hold.wait().await;
4787 }
4788
4789 if self
4794 .forwarding
4795 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
4796 .map_err(RouterError::Forwarding)?
4797 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
4798 {
4799 self.send_abandoned_route_bind_goodbye(
4800 &module_sink,
4801 negotiated_ver,
4802 module_channel,
4803 module_epoch,
4804 );
4805 }
4806
4807 timings.enter(RouteOpenPhase::BindAckWait);
4808 let outcome = timeout_at(relay_deadline, receiver).await;
4809 timings.enter(RouteOpenPhase::Admission);
4810 match outcome {
4811 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
4812 reservation.disarm();
4813 if breaker.record_accepted() {
4814 info!(
4815 module_id = %target_module_id,
4816 "route.bind breaker closed: the probe was accepted"
4817 );
4818 }
4819 self.observe_route_open_accept(ctx, &target_module_id, &principal_label);
4820 Ok(Vec::new())
4821 }
4822 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
4823 reservation.release_and_disarm();
4824 breaker.record_inconclusive();
4828 let scope_code = match body.code.as_str() {
4832 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4833 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4834 _ => None,
4835 };
4836 if let Some(code) = scope_code {
4837 self.observe_route_open_refusal(
4838 ctx,
4839 &target_module_id,
4840 "scope_changed_before_commit",
4841 code,
4842 );
4843 return Ok(vec![control_error_body_frame(&frame, body)?]);
4844 }
4845 self.counters
4846 .increment_route_open_refused("module_rejected");
4847 info!(
4848 target: "control",
4849 code = "module_rejected",
4850 module_code = ?body.code,
4851 module_id = ?target_module_id,
4852 connection_id = ctx.connection_id.get(),
4853 "route.open refused"
4854 );
4855 Ok(vec![control_error_body_frame(&frame, body)?])
4856 }
4857 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
4858 reservation.release_and_disarm();
4859 breaker.record_inconclusive();
4860 tracing::warn!(
4867 module_id = %target_module_id,
4868 "route.bind relay abandoned: {message}"
4869 );
4870 Ok(vec![self.route_open_refusal_frame(
4871 ctx,
4872 &frame,
4873 &target_module_id,
4874 "relay_abandoned",
4875 "target_unavailable",
4876 message,
4877 )?])
4878 }
4879 Ok(Err(_)) => {
4880 reservation.release_and_disarm();
4881 breaker.record_inconclusive();
4882 Ok(vec![self.route_open_refusal_frame(
4883 ctx,
4884 &frame,
4885 &target_module_id,
4886 "relay_waiter_canceled",
4887 "target_unavailable",
4888 "route.bind relay waiter was canceled before the module responded",
4889 )?])
4890 }
4891 Err(_) => {
4892 reservation.release_and_disarm();
4893 if let Some(opened) = breaker.record_timeout(
4897 self.route_bind_breaker_threshold,
4898 self.route_bind_breaker_cooldown,
4899 ) {
4900 warn!(
4901 module_id = %target_module_id,
4902 consecutive_timeouts = opened.consecutive_timeouts,
4903 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
4904 reopened_after_probe = opened.reopened_after_probe,
4905 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
4906 );
4907 }
4908 tracing::warn!(
4916 module_id = %target_module_id,
4917 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
4918 "route.bind relay timed out: module did not ack within budget"
4919 );
4920 Ok(vec![self.route_open_refusal_frame(
4921 ctx,
4922 &frame,
4923 &target_module_id,
4924 "relay_timed_out",
4925 "module_timeout",
4926 format!(
4927 "module_id '{target_module_id}' did not answer route.bind within {:?}",
4928 route_bind_relay_timeout
4929 ),
4930 )?])
4931 }
4932 }
4933 }
4934
4935 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4936 let response = ClientControlResponse::SupervisorSpawnSnapshot {
4937 snapshot: self.supervisor.spawn_snapshot(),
4938 };
4939 Ok(vec![control_response_body_frame(
4940 &frame,
4941 &response,
4942 "ClientControlResponse::SupervisorSpawnSnapshot",
4943 )?])
4944 }
4945
4946 fn handle_supervisor_spawn_subscribe(
4947 &self,
4948 ctx: &RouteCtx,
4949 frame: Frame,
4950 since: Option<SpawnCursor>,
4951 ) -> Result<Vec<Frame>, RouterError> {
4952 match self.supervisor.subscribe_spawns(
4953 ctx.connection_id,
4954 frame.header.corr,
4955 response_version(&frame),
4956 since,
4957 ctx.egress.clone(),
4958 ) {
4959 Ok(()) => Ok(Vec::new()),
4960 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
4961 Ok(vec![control_error_body_frame(
4962 &frame,
4963 ErrorBody {
4964 code: "spawn_cursor_incarnation_mismatch".to_string(),
4965 message: "spawn cursor belongs to a different daemon incarnation"
4966 .to_string(),
4967 detail: Some(serde_json::json!({
4968 "current_daemon_incarnation": current
4969 })),
4970 },
4971 )?])
4972 }
4973 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
4974 &frame,
4975 ErrorBody {
4976 code: "spawn_cursor_too_old".to_string(),
4977 message: "spawn cursor predates the retained event ring".to_string(),
4978 detail: Some(serde_json::json!({
4979 "oldest_retained_cursor": oldest
4980 })),
4981 },
4982 )?]),
4983 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
4984 0,
4985 frame.header.corr,
4986 format!("failed to open supervisor spawn subscription: {error}"),
4987 )),
4988 }
4989 }
4990
4991 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4992 let generation = self
4993 .registry
4994 .generation()
4995 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
4996 let mut modules = Vec::new();
4997 for module in self.supervisor.list() {
4998 let status = module.status_for_control("list").map_err(|err| {
4999 RouterError::backend(
5000 0,
5001 frame.header.corr,
5002 format!("failed to read supervisor status: {err}"),
5003 )
5004 })?;
5005 let (configured, _) = module.configuration().map_err(|err| {
5006 RouterError::backend(
5007 0,
5008 frame.header.corr,
5009 format!("failed to read module configuration: {err}"),
5010 )
5011 })?;
5012 let image = module.running_image_agreement().await;
5014 let resources = Some(module.child_resource_usage());
5017 let mut verdict =
5018 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5019 verdict.spec = Some(launch_spec_agreement(
5020 &configured,
5021 status.process_alive,
5022 status.launch_spec_digest.as_ref(),
5023 ));
5024 let pending_reload = Some(verdict);
5025 modules.push(
5028 SupervisorEntry::new(
5029 status.module_id,
5030 status.state.to_string(),
5031 status.enabled,
5032 status.live,
5033 status.health.status,
5034 )
5035 .with_launch_nonce_env(Some(!cfg!(unix)))
5036 .with_launch_nonce_source(status.launch_nonce_source)
5037 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5038 .with_protocol(status.protocol)
5039 .with_pending_reload(pending_reload)
5040 .with_last_probe_ms(status.health.last_probe_ms)
5041 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5042 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5043 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5044 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5045 .with_restart_count(Some(status.restart_count))
5046 .with_max_restarts(Some(status.max_restarts))
5047 .with_lifetime_restarts(Some(status.lifetime_restarts))
5048 .with_spawn_generation(Some(status.spawn_generation))
5049 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5050 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5051 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5052 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5053 .with_resources(resources),
5054 );
5055 }
5056 let response = ClientControlResponse::SupervisorList {
5057 generation,
5058 modules,
5059 };
5060 Ok(vec![control_response_body_frame(
5061 &frame,
5062 &response,
5063 "ClientControlResponse::SupervisorList",
5064 )?])
5065 }
5066
5067 fn handle_supervisor_stderr_tail(
5068 &self,
5069 frame: Frame,
5070 module_id: String,
5071 max_lines: Option<u32>,
5072 max_bytes: Option<u32>,
5073 ) -> Result<Vec<Frame>, RouterError> {
5074 let Some(module) = self.supervisor.get(&module_id) else {
5075 return Ok(vec![control_error_frame(
5076 &frame,
5077 "unknown_module",
5078 format!("module_id '{module_id}' is not supervised"),
5079 )?]);
5080 };
5081
5082 let snapshot = module.stderr_tail(
5083 max_lines.map(|value| value as usize),
5084 max_bytes.map(|value| value as usize),
5085 );
5086
5087 let response = ClientControlResponse::SupervisorStderrTail {
5088 module_id,
5089 tail: StderrTail {
5090 capture: match snapshot.capture {
5091 CaptureState::Captured => StderrCaptureState::Captured,
5092 CaptureState::Incomplete { reason } => {
5093 StderrCaptureState::Incomplete { reason }
5094 }
5095 CaptureState::NotCaptured { reason } => {
5096 StderrCaptureState::NotCaptured { reason }
5097 }
5098 },
5099 entries: snapshot
5100 .entries
5101 .into_iter()
5102 .map(|entry| match entry {
5103 TailEntry::Line {
5104 text,
5105 truncated,
5106 at_ms,
5107 } => StderrTailEntry::Line {
5108 text,
5109 truncated,
5110 at_ms,
5111 },
5112 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5113 })
5114 .collect(),
5115 dropped_lines: snapshot.dropped_lines,
5116 },
5117 };
5118 Ok(vec![control_response_body_frame(
5119 &frame,
5120 &response,
5121 "ClientControlResponse::SupervisorStderrTail",
5122 )?])
5123 }
5124
5125 async fn handle_supervisor_terminals(
5126 &self,
5127 frame: Frame,
5128 module_id: String,
5129 ) -> Result<Vec<Frame>, RouterError> {
5130 let Some(module) = self.supervisor.get(&module_id) else {
5131 return Ok(vec![control_error_frame(
5132 &frame,
5133 "unknown_module",
5134 format!("module_id '{module_id}' is not supervised"),
5135 )?]);
5136 };
5137
5138 let terminals = module
5141 .read_durable_terminal_history()
5142 .await
5143 .map_err(|error| {
5144 RouterError::backend(
5145 0,
5146 frame.header.corr,
5147 format!("failed to read terminal history: {error}"),
5148 )
5149 })?;
5150 let response = ClientControlResponse::SupervisorTerminals {
5151 module_id,
5152 terminals,
5153 };
5154 Ok(vec![control_response_body_frame(
5155 &frame,
5156 &response,
5157 "ClientControlResponse::SupervisorTerminals",
5158 )?])
5159 }
5160
5161 fn handle_supervisor_routes(
5162 &self,
5163 frame: Frame,
5164 module_id: Option<String>,
5165 ) -> Result<Vec<Frame>, RouterError> {
5166 let modules = self
5167 .forwarding
5168 .route_census(module_id.as_deref())
5169 .map_err(RouterError::Forwarding)?
5170 .into_iter()
5171 .map(|(module_id, routes)| SupervisorRouteModule {
5172 module_id,
5173 routes: routes
5174 .into_iter()
5175 .map(|route| SupervisorRoute {
5176 consumer: match route.principal {
5177 Principal::Reserved { module_id } => {
5178 SupervisorRouteConsumer::Reserved { module_id }
5179 }
5180 Principal::Direct | Principal::Unverified => {
5181 SupervisorRouteConsumer::Direct {
5182 connection_id: route.goodbye_target.connection_id.get(),
5183 }
5184 }
5185 },
5186 age_ms: Instant::now()
5187 .saturating_duration_since(route.bound_at)
5188 .as_millis()
5189 .try_into()
5190 .unwrap_or(u64::MAX),
5191 draining: route.draining,
5192 drain_reason: route.drain_reason,
5193 })
5194 .collect(),
5195 })
5196 .collect();
5197 let response = ClientControlResponse::SupervisorRoutes { modules };
5198 Ok(vec![control_response_body_frame(
5199 &frame,
5200 &response,
5201 "ClientControlResponse::SupervisorRoutes",
5202 )?])
5203 }
5204
5205 async fn handle_supervisor_provenance(
5206 &self,
5207 frame: Frame,
5208 module_id: Option<String>,
5209 ) -> Result<Vec<Frame>, RouterError> {
5210 let mut selected = if let Some(module_id) = module_id {
5211 let Some(module) = self.supervisor.get(&module_id) else {
5212 return Ok(vec![control_error_frame(
5213 &frame,
5214 "unknown_module",
5215 format!("module_id '{module_id}' is not supervised"),
5216 )?]);
5217 };
5218 vec![module]
5219 } else {
5220 self.supervisor.list()
5221 };
5222
5223 let mut modules = Vec::with_capacity(selected.len());
5224 for module in selected.drain(..) {
5225 let (status, observed_image) = module
5226 .status_and_running_image_agreement()
5227 .await
5228 .map_err(|err| {
5229 RouterError::backend(
5230 0,
5231 frame.header.corr,
5232 format!("failed to read supervisor status: {err}"),
5233 )
5234 })?;
5235 let module_declared = self
5236 .registry
5237 .get_module(&status.module_id)
5238 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5239 .and_then(|registration| registration.manifest.provenance)
5240 .map(|build| ModuleDeclaredProvenance::Reported { build })
5241 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5242 #[cfg(test)]
5243 let running_image = match &self.provenance_probe_override {
5244 Some(result) => result.clone(),
5245 None => observed_image,
5246 };
5247 #[cfg(not(test))]
5248 let running_image = observed_image;
5249 modules.push(SupervisorModuleProvenance {
5250 module_id: status.module_id,
5251 module_declared,
5252 daemon_observed: SupervisorObservedProcess {
5253 launch_nonce_source: status.launch_nonce_source,
5254 pid: status.pid,
5255 spawned_at_ms: status.spawned_at_ms,
5256 spawned_from: status.spawned_from,
5257 running_image,
5258 },
5259 });
5260 }
5261 let daemon = SupervisorDaemonProvenance {
5262 daemon_build: self.daemon_provenance.build.clone(),
5263 daemon_observed: DaemonObservedProcess {
5264 pid: self.daemon_provenance.pid,
5265 started_at_ms: self
5266 .daemon_provenance
5267 .start_clock
5268 .map(|clock| clock.started_at_ms())
5269 .or(self.daemon_provenance.started_at_ms),
5270 running_image: self
5271 .daemon_provenance
5272 .probe
5273 .observe(
5274 self.daemon_provenance.pid,
5275 self.daemon_provenance.executable_path.as_deref(),
5276 self.daemon_provenance.executable_identity.clone(),
5277 self.daemon_provenance.process_start_time,
5278 )
5279 .await,
5280 },
5281 };
5282 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5283 Ok(vec![control_response_body_frame(
5284 &frame,
5285 &response,
5286 "ClientControlResponse::SupervisorProvenance",
5287 )?])
5288 }
5289
5290 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5291 self.refresh_capability_requirements();
5292 let generation = self
5293 .registry
5294 .generation()
5295 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5296 let modules = self
5297 .supervisor
5298 .list()
5299 .into_iter()
5300 .map(|module| {
5301 let status = module.status_for_control("health").map_err(|err| {
5302 RouterError::backend(
5303 0,
5304 frame.header.corr,
5305 format!("failed to read supervisor health: {err}"),
5306 )
5307 })?;
5308 let module_id = status.module_id;
5309 let capability_detail = self
5310 .capability_evaluator
5311 .required_problem_detail(&module_id);
5312 Ok(SupervisorHealthEntry {
5313 module_id,
5314 status: status.health.status,
5315 detail: append_capability_problem_detail(
5316 status.health.detail,
5317 capability_detail,
5318 ),
5319 metrics: status.health.metrics,
5320 consecutive_failures: status.health.consecutive_failures,
5321 late_answer_count: status.health.late_answer_count,
5322 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5323 last_action: status.health.last_action,
5324 last_action_ms: status.health.last_action_ms,
5325 last_probe_ms: status.health.last_probe_ms,
5326 })
5327 })
5328 .collect::<Result<Vec<_>, RouterError>>()?;
5329 let response = ClientControlResponse::SupervisorHealth {
5330 generation,
5331 modules,
5332 };
5333 Ok(vec![control_response_body_frame(
5334 &frame,
5335 &response,
5336 "ClientControlResponse::SupervisorHealth",
5337 )?])
5338 }
5339
5340 async fn handle_supervisor_restart(
5341 &self,
5342 frame: Frame,
5343 module_id: String,
5344 drain_timeout_ms: Option<u64>,
5345 requester: Principal,
5346 ) -> Result<Vec<Frame>, RouterError> {
5347 let operation_lock = self.supervisor.operation_lock();
5348 let _operation_guard = operation_lock.lock().await;
5349 let Some(module) = self.supervisor.get(&module_id) else {
5350 return Ok(vec![control_error_frame(
5351 &frame,
5352 "unknown_module",
5353 format!("module_id '{module_id}' is not supervised"),
5354 )?]);
5355 };
5356
5357 self.route_outages.mark_operator_action(&module_id);
5358 if let Err(err) = module
5359 .restart_requested(drain_timeout_ms, Some(requester))
5360 .await
5361 {
5362 self.route_outages
5363 .operator_action_ended_unrefused(&module_id);
5364 let (code, message) = match err {
5365 crate::supervise::SuperviseError::Disabled { .. } => {
5366 ("module_disabled", err.to_string())
5367 }
5368 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5369 ("swap_in_progress", err.to_string())
5370 }
5371 _ => (
5372 "target_unavailable",
5373 format!("failed to restart module_id '{module_id}': {err}"),
5374 ),
5375 };
5376 return Ok(vec![control_error_frame(&frame, code, message)?]);
5377 }
5378
5379 let response = ClientControlResponse::SupervisorAck {
5380 module_id,
5381 applied: true,
5382 };
5383 Ok(vec![control_response_body_frame(
5384 &frame,
5385 &response,
5386 "ClientControlResponse::SupervisorAck",
5387 )?])
5388 }
5389
5390 async fn handle_supervisor_swap(
5394 &self,
5395 frame: Frame,
5396 module_id: String,
5397 ready_timeout_ms: Option<u64>,
5398 requester: Principal,
5399 ) -> Result<Vec<Frame>, RouterError> {
5400 let module = {
5407 let operation_lock = self.supervisor.operation_lock();
5408 let _operation_guard = operation_lock.lock().await;
5409 self.supervisor.get(&module_id)
5410 };
5411 let Some(module) = module else {
5412 return Ok(vec![control_error_frame(
5413 &frame,
5414 "unknown_module",
5415 format!("module_id '{module_id}' is not supervised"),
5416 )?]);
5417 };
5418
5419 self.route_outages.mark_operator_action(&module_id);
5420 if let Err(err) = module
5421 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
5422 .await
5423 {
5424 self.route_outages
5425 .operator_action_ended_unrefused(&module_id);
5426 use crate::supervise::SuperviseError;
5427 let message = err.to_string();
5428 let error = match err {
5429 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
5430 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
5431 code: "swap_refused".to_string(),
5432 message,
5433 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
5434 },
5435 SuperviseError::SwapFailed {
5436 arm,
5437 candidate_exit,
5438 ..
5439 } => ErrorBody {
5440 code: "swap_failed".to_string(),
5441 message,
5442 detail: Some(serde_json::json!({
5443 "arm": arm.as_str(),
5444 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
5445 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
5446 })),
5447 },
5448 _ => ErrorBody::new(
5449 "target_unavailable",
5450 format!("failed to swap module_id '{module_id}': {message}"),
5451 ),
5452 };
5453 return Ok(vec![control_error_body_frame(&frame, error)?]);
5454 }
5455 self.route_outages
5459 .operator_action_ended_unrefused(&module_id);
5460
5461 let response = ClientControlResponse::SupervisorAck {
5462 module_id,
5463 applied: true,
5464 };
5465 Ok(vec![control_response_body_frame(
5466 &frame,
5467 &response,
5468 "ClientControlResponse::SupervisorAck",
5469 )?])
5470 }
5471
5472 async fn handle_supervisor_reload(
5473 &self,
5474 frame: Frame,
5475 module_id: String,
5476 requester: Principal,
5477 ) -> Result<Vec<Frame>, RouterError> {
5478 let operation_lock = self.supervisor.operation_lock();
5479 let _operation_guard = operation_lock.lock().await;
5480 let Some(module) = self.supervisor.get(&module_id) else {
5481 return Ok(vec![control_error_frame(
5482 &frame,
5483 "unknown_module",
5484 format!("module_id '{module_id}' is not supervised"),
5485 )?]);
5486 };
5487
5488 self.route_outages.mark_operator_action(&module_id);
5489 if let Err(err) = module.reload_requested(Some(requester)).await {
5490 self.route_outages
5491 .operator_action_ended_unrefused(&module_id);
5492 let (code, message) = match err {
5493 crate::supervise::SuperviseError::Disabled { .. } => {
5494 ("module_disabled", err.to_string())
5495 }
5496 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5497 ("swap_in_progress", err.to_string())
5498 }
5499 _ => (
5500 "reload_failed",
5501 format!("failed to reload module_id '{module_id}': {err}"),
5502 ),
5503 };
5504 return Ok(vec![control_error_frame(&frame, code, message)?]);
5505 }
5506
5507 let response = ClientControlResponse::SupervisorAck {
5508 module_id,
5509 applied: true,
5510 };
5511 Ok(vec![control_response_body_frame(
5512 &frame,
5513 &response,
5514 "ClientControlResponse::SupervisorAck",
5515 )?])
5516 }
5517
5518 async fn handle_supervisor_rescan(
5519 &self,
5520 frame: Frame,
5521 preview: bool,
5522 requester: Principal,
5523 ) -> Result<Vec<Frame>, RouterError> {
5524 let Some(context) = self.rescan.clone() else {
5525 return Ok(vec![control_error_frame(
5526 &frame,
5527 "rescan_unavailable",
5528 "the daemon was not started with a reloadable config path".to_string(),
5529 )?]);
5530 };
5531
5532 let operation_lock = self.supervisor.operation_lock();
5533 let _operation_guard = operation_lock.lock().await;
5534 let loaded = match crate::daemon_config::load_with_activity(&context.config_path) {
5535 Ok(config) => config,
5536 Err(err) => {
5537 return Ok(vec![control_error_frame(
5538 &frame,
5539 "invalid_daemon_config",
5540 format!("supervisor rescan rejected daemon config: {err}"),
5541 )?])
5542 }
5543 };
5544 let Some((config, activity_config)) = loaded else {
5552 return Ok(vec![control_error_frame(
5553 &frame,
5554 "invalid_daemon_config",
5555 format!(
5556 "daemon config not found at {}; refusing to rescan (an absent config would \
5557 retire every supervised module)",
5558 context.config_path.display()
5559 ),
5560 )?]);
5561 };
5562 let (
5563 configured_port,
5564 storage_config,
5565 admission_facts_carrier_module_id,
5566 admission_facts_targets,
5567 scope_authority_owners,
5568 privileged_operator_prompt,
5569 modules,
5570 reserved_capabilities,
5571 ) = (
5572 config.port,
5573 config.storage,
5574 config.admission_facts_carrier_module_id,
5575 config.admission_facts_targets,
5576 config.scope_authority_owners,
5577 config.privileged_operator_prompt,
5578 config.modules,
5579 config.reserved_capabilities,
5580 );
5581
5582 let mut restart_required = Vec::new();
5591 for section in RestartRequiredSection::ALL {
5592 let changed = match section {
5593 RestartRequiredSection::Port => configured_port != context.configured_port,
5594 RestartRequiredSection::Storage => storage_config != context.storage_config,
5595 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
5596 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
5597 }
5598 RestartRequiredSection::AdmissionFactsTargets => {
5599 admission_facts_targets != context.admission_facts_targets
5600 }
5601 RestartRequiredSection::ScopeAuthorityOwners => {
5602 scope_authority_owners != context.scope_authority_owners
5603 }
5604 RestartRequiredSection::PrivilegedOperatorPrompt => {
5605 privileged_operator_prompt != context.privileged_operator_prompt
5606 }
5607 };
5608 if changed {
5609 restart_required.push(section.label().to_string());
5610 }
5611 }
5612 if !restart_required.is_empty() {
5613 warn!(
5614 config_path = %context.config_path.display(),
5615 sections = %restart_required.join(", "),
5616 "daemon config changed outside the modules section; restart the daemon to apply those changes"
5617 );
5618 }
5619
5620 for configured in &modules {
5621 if let Err(err) = validate_spec(&configured.module_spec()) {
5622 return Ok(vec![control_error_frame(
5623 &frame,
5624 "invalid_daemon_config",
5625 format!("supervisor rescan rejected daemon config: {err}"),
5626 )?]);
5627 }
5628 }
5629
5630 let configured_capabilities = modules
5631 .iter()
5632 .map(|module| (module.module_id.clone(), module.enabled))
5633 .collect::<Vec<_>>();
5634 let preview_capability_warnings = if preview {
5635 let (_, registrations) = self.runtime_capability_snapshot()?;
5636 let current_modules = self
5637 .supervisor
5638 .list()
5639 .into_iter()
5640 .map(|module| module.module_id().to_string())
5641 .collect::<BTreeSet<_>>();
5642 let resulting_modules = configured_capabilities.clone();
5643 let removed = current_modules
5644 .into_iter()
5645 .filter(|module_id| {
5646 !resulting_modules
5647 .iter()
5648 .any(|(configured_id, _)| configured_id == module_id)
5649 })
5650 .collect::<Vec<_>>();
5651 self.capability_evaluator.preview_removal_warnings(
5652 resulting_modules,
5653 &removed,
5654 ®istrations,
5655 )
5656 } else {
5657 Vec::new()
5658 };
5659 let result = match self
5660 .reconcile_supervised_modules(&context.supervisor, modules, preview, requester.clone())
5661 .await
5662 {
5663 Ok(result) => result,
5664 Err(message) => {
5665 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
5666 }
5667 };
5668 if !preview {
5669 if let Some(ledger) = &self.activity {
5670 ledger.module_rescanned(&result.added, &result.removed, requester);
5671 }
5672 if let Err(code) = self.configure_activity(activity_config).await {
5673 warn!(
5674 code,
5675 "activity retention could not be applied during rescan"
5676 );
5677 }
5678 self.capability_evaluator
5679 .configure(configured_capabilities, reserved_capabilities);
5680 self.capability_evaluator.wake_deadline_loop();
5681 self.refresh_capability_requirements();
5682 }
5683 let mut result = result;
5684 result.restart_required = restart_required;
5685 result.capability_warnings = preview_capability_warnings;
5686 let response = ClientControlResponse::SupervisorRescan { result };
5687 Ok(vec![control_response_body_frame(
5688 &frame,
5689 &response,
5690 "ClientControlResponse::SupervisorRescan",
5691 )?])
5692 }
5693
5694 async fn handle_supervisor_release_reserved(
5695 &self,
5696 frame: Frame,
5697 module_id: String,
5698 ) -> Result<Vec<Frame>, RouterError> {
5699 let Some(context) = self.rescan.clone() else {
5700 return Ok(vec![control_error_frame(
5701 &frame,
5702 "release_unavailable",
5703 "reserved-id release requires a daemon started with a reloadable config path",
5704 )?]);
5705 };
5706 let operation_lock = self.supervisor.operation_lock();
5707 let _operation_guard = operation_lock.lock().await;
5708 let loaded = match crate::daemon_config::load(&context.config_path) {
5709 Ok(Some(config)) => config,
5710 Ok(None) => {
5711 return Ok(vec![control_error_frame(
5712 &frame,
5713 "invalid_daemon_config",
5714 format!(
5715 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
5716 context.config_path.display()
5717 ),
5718 )?])
5719 }
5720 Err(err) => {
5721 return Ok(vec![control_error_frame(
5722 &frame,
5723 "invalid_daemon_config",
5724 format!("unable to verify reserved-id release against daemon config: {err}"),
5725 )?])
5726 }
5727 };
5728 if loaded
5729 .modules
5730 .iter()
5731 .any(|configured| configured.module_id == module_id)
5732 {
5733 return Ok(vec![control_error_frame(
5734 &frame,
5735 "reserved_module_configured",
5736 format!(
5737 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
5738 ),
5739 )?]);
5740 }
5741 if !self.supervisor.release_retained_reserved_gate(&module_id) {
5742 return Ok(vec![control_error_frame(
5743 &frame,
5744 "reserved_gate_not_retained",
5745 format!(
5746 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
5747 ),
5748 )?]);
5749 }
5750
5751 let response = ClientControlResponse::SupervisorAck {
5752 module_id,
5753 applied: true,
5754 };
5755 Ok(vec![control_response_body_frame(
5756 &frame,
5757 &response,
5758 "ClientControlResponse::SupervisorAck",
5759 )?])
5760 }
5761
5762 async fn reconcile_supervised_modules(
5771 &self,
5772 supervisor: &Supervisor,
5773 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
5774 preview: bool,
5775 requester: Principal,
5776 ) -> Result<SupervisorRescanResult, String> {
5777 let mut current = BTreeMap::new();
5778 for module in self.supervisor.list() {
5779 let (spec, health) = module.configuration().map_err(|err| {
5780 format!(
5781 "failed to read configuration for module_id '{}': {err}",
5782 module.module_id()
5783 )
5784 })?;
5785 let enabled = module
5786 .status()
5787 .map_err(|err| {
5788 format!(
5789 "failed to read status for module_id '{}': {err}",
5790 module.module_id()
5791 )
5792 })?
5793 .enabled;
5794 current.insert(
5795 module.module_id().to_string(),
5796 (module, spec, health, enabled),
5797 );
5798 }
5799 let configured = configured_modules
5800 .into_iter()
5801 .map(|module| (module.module_id.clone(), module))
5802 .collect::<BTreeMap<_, _>>();
5803
5804 let added = configured
5805 .keys()
5806 .filter(|module_id| !current.contains_key(*module_id))
5807 .cloned()
5808 .collect::<Vec<_>>();
5809 let removed = current
5810 .keys()
5811 .filter(|module_id| !configured.contains_key(*module_id))
5812 .cloned()
5813 .collect::<Vec<_>>();
5814 let mut changed_pending_reload = Vec::new();
5815 let mut configuration_changes = BTreeSet::new();
5816 let mut enabled_changes = BTreeSet::new();
5817 let mut unchanged = 0_u32;
5818
5819 for (module_id, configured_module) in &configured {
5820 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
5821 else {
5822 continue;
5823 };
5824 let launch_changed = *current_spec != configured_module.module_spec();
5828 let configuration_changed =
5829 launch_changed || *current_health != configured_module.health;
5830 let enabled_changed = *current_enabled != configured_module.enabled;
5831 if configuration_changed {
5832 configuration_changes.insert(module_id.clone());
5833 }
5834 if launch_changed {
5835 changed_pending_reload.push(module_id.clone());
5836 }
5837 if enabled_changed {
5838 enabled_changes.insert(module_id.clone());
5839 }
5840 if !configuration_changed && !enabled_changed {
5841 unchanged = unchanged.saturating_add(1);
5842 }
5843 }
5844
5845 if preview {
5851 return Ok(SupervisorRescanResult {
5852 added,
5853 removed,
5854 changed_pending_reload,
5855 enabled_changes: enabled_changes.iter().cloned().collect(),
5856 unchanged,
5857 preview: true,
5858 restart_required: Vec::new(),
5862 capability_warnings: Vec::new(),
5863 });
5864 }
5865
5866 for module_id in &removed {
5867 let module = ¤t
5868 .get(module_id)
5869 .expect("removed module came from current supervisor state")
5870 .0;
5871 module
5872 .retire_requested(Some(requester.clone()))
5873 .await
5874 .map_err(|err| {
5875 format!("failed to retire module_id '{module_id}' during rescan: {err}")
5876 })?;
5877 self.supervisor.record_rescan_removal(module_id);
5911 self.supervisor.retire(module_id);
5912 self.route_outages.forget(module_id);
5913 }
5914
5915 for module_id in configured.keys() {
5916 let Some((module, _, _, _)) = current.get(module_id) else {
5917 continue;
5918 };
5919 let configured_module = configured
5920 .get(module_id)
5921 .expect("configured module id came from configured map");
5922 if configuration_changes.contains(module_id) {
5923 module
5924 .update_configuration(
5925 configured_module.module_spec(),
5926 configured_module.health.clone(),
5927 configured_module.drain_timeout_ms,
5928 )
5929 .await
5930 .map_err(|err| {
5931 format!(
5932 "failed to update module_id '{module_id}' configuration during rescan: {err}"
5933 )
5934 })?;
5935 }
5936 if enabled_changes.contains(module_id) {
5937 self.route_outages.mark_operator_action(module_id);
5940 module
5941 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
5942 .await
5943 .map_err(|err| {
5944 self.route_outages.operator_action_ended_unrefused(module_id);
5945 format!(
5946 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
5947 configured_module.enabled
5948 )
5949 })?;
5950 }
5951 }
5952
5953 for module_id in &added {
5954 let configured_module = configured
5955 .get(module_id)
5956 .expect("added module id came from configured map");
5957 supervisor
5958 .supervise_configured_requested(
5959 configured_module.module_spec(),
5960 configured_module.enabled,
5961 configured_module.health.clone(),
5962 configured_module.drain_timeout_ms,
5963 configured_module.restart,
5964 Some(requester.clone()),
5965 )
5966 .map_err(|err| {
5967 format!("failed to add module_id '{module_id}' during rescan: {err}")
5968 })?;
5969 }
5970
5971 Ok(SupervisorRescanResult {
5972 added,
5973 removed,
5974 changed_pending_reload,
5975 enabled_changes: enabled_changes.iter().cloned().collect(),
5976 unchanged,
5977 preview: false,
5978 restart_required: Vec::new(),
5981 capability_warnings: Vec::new(),
5982 })
5983 }
5984
5985 async fn handle_supervisor_set_enabled(
5986 &self,
5987 frame: Frame,
5988 module_id: String,
5989 enabled: bool,
5990 requester: Principal,
5991 ) -> Result<Vec<Frame>, RouterError> {
5992 let operation_lock = self.supervisor.operation_lock();
5993 let _operation_guard = operation_lock.lock().await;
5994 let Some(module) = self.supervisor.get(&module_id) else {
5995 return Ok(vec![control_error_frame(
5996 &frame,
5997 "unknown_module",
5998 format!("module_id '{module_id}' is not supervised"),
5999 )?]);
6000 };
6001
6002 self.route_outages.mark_operator_action(&module_id);
6005 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6006 Ok(applied) => applied,
6007 Err(err) => {
6008 self.route_outages
6009 .operator_action_ended_unrefused(&module_id);
6010 return Ok(vec![control_error_frame(
6011 &frame,
6012 "target_unavailable",
6013 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6014 )?]);
6015 }
6016 };
6017 if !applied {
6018 self.route_outages
6021 .operator_action_ended_unrefused(&module_id);
6022 }
6023
6024 self.capability_evaluator.wake_deadline_loop();
6025 self.refresh_capability_requirements();
6026 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6027 Ok(vec![control_response_body_frame(
6028 &frame,
6029 &response,
6030 "ClientControlResponse::SupervisorAck",
6031 )?])
6032 }
6033
6034 async fn handle_supervisor_health_probe(
6035 &self,
6036 frame: Frame,
6037 module_id: String,
6038 ) -> Result<Vec<Frame>, RouterError> {
6039 self.refresh_capability_requirements();
6040 let Some(registration) = self
6041 .registry
6042 .get_module(&module_id)
6043 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6044 else {
6045 return Ok(vec![control_error_frame(
6046 &frame,
6047 "unknown_module",
6048 format!("module_id '{module_id}' is not registered"),
6049 )?]);
6050 };
6051
6052 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6065 {
6066 return Ok(vec![control_error_frame(
6067 &frame,
6068 "health_not_advertised",
6069 format!("module_id '{module_id}' did not advertise health.check"),
6070 )?]);
6071 }
6072
6073 let deadline = Instant::now() + self.health_probe_timeout;
6074 let pending = match self.forwarding.begin_module_control_rpc_for(
6075 &module_id,
6076 MODULE_CONTROL_OP_HEALTH_CHECK,
6077 deadline,
6078 ) {
6079 Ok(pending) => pending,
6080 Err(err) => {
6081 return Ok(vec![control_error_frame(
6082 &frame,
6083 forwarding_error_code(&err),
6084 err.to_string(),
6085 )?])
6086 }
6087 };
6088
6089 let PendingModuleControlRpc {
6090 endpoint,
6091 module_sink,
6092 negotiated_ver,
6093 corr: probe_corr,
6094 receiver,
6095 } = pending;
6096 let mut guard =
6097 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6098 let probe_body =
6099 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6100 RouterError::backend(
6101 0,
6102 frame.header.corr,
6103 format!("failed to encode health.check request: {err}"),
6104 )
6105 })?;
6106 let probe_frame = Frame::build_with_version(
6107 negotiated_ver,
6108 FrameType::Request,
6109 control_flags(),
6110 0,
6111 0,
6112 probe_corr,
6113 probe_body,
6114 )
6115 .map_err(RouterError::FrameBuild)?;
6116
6117 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6120 Ok(Ok(())) => {}
6121 Ok(Err(err)) => {
6122 return Ok(vec![control_error_frame(
6123 &frame,
6124 "target_unavailable",
6125 err.to_string(),
6126 )?]);
6127 }
6128 Err(_) => {
6129 return Ok(vec![control_error_frame(
6130 &frame,
6131 "module_timeout",
6132 format!(
6133 "module_id '{module_id}' did not answer health.check within {:?}",
6134 self.health_probe_timeout
6135 ),
6136 )?]);
6137 }
6138 }
6139
6140 match timeout_at(deadline, receiver).await {
6141 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6142 guard.disarm();
6143 let Some(report) = response.health_report() else {
6144 return Ok(vec![control_error_frame(
6145 &frame,
6146 "invalid_control_body",
6147 "health.check RPC returned a non-health response",
6148 )?]);
6149 };
6150 let HealthReport {
6155 status,
6156 detail,
6157 metrics,
6158 } = report;
6159 let capability_detail = self
6160 .capability_evaluator
6161 .required_problem_detail(&module_id);
6162 let response = ClientControlResponse::SupervisorHealthProbe {
6163 module_id,
6164 status,
6165 detail: append_capability_problem_detail(detail, capability_detail),
6166 metrics,
6167 };
6168 Ok(vec![control_response_body_frame(
6169 &frame,
6170 &response,
6171 "ClientControlResponse::SupervisorHealthProbe",
6172 )?])
6173 }
6174 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6175 guard.disarm();
6176 Ok(vec![control_error_body_frame(&frame, body)?])
6177 }
6178 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6179 guard.disarm();
6180 Ok(vec![control_error_frame(
6181 &frame,
6182 "target_unavailable",
6183 message,
6184 )?])
6185 }
6186 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6187 guard.disarm();
6188 Ok(vec![control_error_frame(
6189 &frame,
6190 "invalid_control_body",
6191 message,
6192 )?])
6193 }
6194 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6195 guard.disarm();
6196 Ok(vec![control_error_frame(
6197 &frame,
6198 "invalid_control_body",
6199 format!("expected module-control op '{expected}', got '{actual}'"),
6200 )?])
6201 }
6202 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6203 guard.disarm();
6204 Ok(vec![control_error_frame(
6205 &frame,
6206 "module_timeout",
6207 format!(
6208 "module_id '{module_id}' answered health.check after {:?}",
6209 self.health_probe_timeout
6210 ),
6211 )?])
6212 }
6213 Ok(Err(_)) => Ok(vec![control_error_frame(
6214 &frame,
6215 "target_unavailable",
6216 "health.check waiter was canceled before the module responded",
6217 )?]),
6218 Err(_) => Ok(vec![control_error_frame(
6219 &frame,
6220 "module_timeout",
6221 format!(
6222 "module_id '{module_id}' did not answer health.check within {:?}",
6223 self.health_probe_timeout
6224 ),
6225 )?]),
6226 }
6227 }
6228
6229 fn supervisor_status(
6230 &self,
6231 module_id: &str,
6232 corr: u64,
6233 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6234 self.supervisor
6235 .get(module_id)
6236 .map(|module| {
6237 let warming = module.is_warming_for_control("status").map_err(|err| {
6238 RouterError::backend(
6239 0,
6240 corr,
6241 format!(
6242 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6243 ),
6244 )
6245 })?;
6246 module.status_for_control("status").map_err(|err| {
6247 RouterError::backend(
6248 0,
6249 corr,
6250 format!(
6251 "failed to read supervisor status for module_id '{module_id}': {err}"
6252 ),
6253 )
6254 }).map(|status| (status, warming))
6255 })
6256 .transpose()
6257 }
6258
6259 fn guard_module_control_op(
6260 &self,
6261 frame: &Frame,
6262 module_id: &str,
6263 op: &str,
6264 ) -> Result<Option<Frame>, RouterError> {
6265 if self.module_grants_op(module_id, op, frame.header.corr)? {
6266 return Ok(None);
6267 }
6268
6269 Ok(Some(control_error_frame(
6270 frame,
6271 "op_not_allowed",
6272 format!("module_id '{module_id}' did not grant control op '{op}'"),
6273 )?))
6274 }
6275
6276 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6277 let Some(registration) = self
6278 .registry
6279 .get_module(module_id)
6280 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6281 else {
6282 return Ok(false);
6283 };
6284 Ok(module_registration_grants_op(®istration.control_ops, op))
6285 }
6286
6287 fn handle_status_update(
6288 &self,
6289 endpoint: ModuleEndpointId,
6290 frame: Frame,
6291 ) -> Result<Vec<Frame>, RouterError> {
6292 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6293 Ok(update) => update,
6294 Err(err) => {
6295 if is_known_module_push_op(&frame.body) {
6300 return Ok(vec![control_error_frame(
6301 &frame,
6302 "invalid_control_body",
6303 format!("malformed module control push body: {err}"),
6304 )?]);
6305 }
6306 return Ok(Vec::new());
6307 }
6308 };
6309
6310 match update {
6311 ModuleControlPush::RouteStatus {
6312 route_channel,
6313 route_epoch,
6314 status,
6315 } => {
6316 self.forwarding
6317 .cache_status(endpoint, route_channel, route_epoch, status)
6318 .map_err(RouterError::Forwarding)?;
6319 }
6320 }
6321 Ok(Vec::new())
6322 }
6323
6324 fn handle_route_poll(
6325 &self,
6326 ctx: &RouteCtx,
6327 frame: Frame,
6328 route_channel: u16,
6329 route_epoch: u32,
6330 kind: PollKind,
6331 ) -> Result<Vec<Frame>, RouterError> {
6332 let snapshot = self
6333 .forwarding
6334 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
6335 .map_err(RouterError::Forwarding)?;
6336 let response = match (kind, snapshot) {
6337 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
6338 ClientControlResponse::RoutePoll {
6339 route_channel,
6340 route_epoch,
6341 status,
6342 live: None,
6343 }
6344 }
6345 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6346 route_channel,
6347 route_epoch,
6348 status: None,
6349 live: None,
6350 },
6351 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
6352 let live = self
6378 .process_liveness
6379 .as_ref()
6380 .and_then(|source| source.process_live(&module_id))
6381 .unwrap_or(true);
6382 ClientControlResponse::RoutePoll {
6383 route_channel,
6384 route_epoch,
6385 status: None,
6386 live: Some(live),
6387 }
6388 }
6389 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
6390 route_channel,
6391 route_epoch,
6392 status: None,
6393 live: Some(false),
6394 },
6395 };
6396
6397 Ok(vec![control_response_body_frame(
6398 &frame,
6399 &response,
6400 "ClientControlResponse::RoutePoll",
6401 )?])
6402 }
6403
6404 pub(crate) fn observe_module_control_completion(
6405 &self,
6406 completion: ModuleControlRpcCompletion,
6407 ) -> bool {
6408 match completion {
6409 ModuleControlRpcCompletion::Unknown => false,
6410 ModuleControlRpcCompletion::Settled => true,
6411 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
6412 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
6413 info!(
6414 module_id = %module_id,
6415 latency_ms,
6416 "late health.check answer proves the module is alive"
6417 );
6418 match self
6419 .supervisor
6420 .record_late_health_answer(&module_id, latency_ms)
6421 {
6422 Ok(true) => {}
6423 Ok(false) => debug!(
6424 module_id = %module_id,
6425 latency_ms,
6426 "late health.check answer has no active supervisor snapshot"
6427 ),
6428 Err(err) => warn!(
6429 module_id = %module_id,
6430 latency_ms,
6431 error = %err,
6432 "failed to record late health.check answer"
6433 ),
6434 }
6435 true
6436 }
6437 }
6438 }
6439
6440 fn refuse_to_end_module_connection_for_a_client(
6461 &self,
6462 module_connection_id: ConnectionId,
6463 corr: u64,
6464 err: ForwardingError,
6465 ) -> Result<(), RouterError> {
6466 if let ForwardingError::ConnectionClosing { connection_id } = err {
6467 if connection_id != module_connection_id {
6468 warn!(
6469 module_connection_id = module_connection_id.get(),
6470 client_connection_id = connection_id.get(),
6471 corr,
6472 "dropping a route.bind response for a closing client; the module connection keeps serving"
6473 );
6474 return Ok(());
6475 }
6476 }
6477 Err(RouterError::Forwarding(err))
6478 }
6479
6480 fn handle_module_relay_response(
6481 &self,
6482 connection_id: ConnectionId,
6483 frame: Frame,
6484 ) -> Result<Vec<Frame>, RouterError> {
6485 let mut secondary_error = None;
6486 let outcome = match frame.header.ty {
6487 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
6488 Ok(probe) if probe.op == "route.bind" => {
6489 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
6490 Ok(ModuleControlResponse::RouteBindAck {}) => {
6491 RouteBindRelayOutcome::Accepted
6492 }
6493 Ok(other) => {
6494 let message =
6495 format!("route.bind response carried unexpected body: {other:?}");
6496 secondary_error = Some(control_error_frame(
6497 &frame,
6498 "invalid_control_body",
6499 message.clone(),
6500 )?);
6501 RouteBindRelayOutcome::ModuleGone(message)
6502 }
6503 Err(err) => {
6504 let message = format!("malformed route.bind response body: {err}");
6505 secondary_error = Some(control_error_frame(
6506 &frame,
6507 "invalid_control_body",
6508 message.clone(),
6509 )?);
6510 RouteBindRelayOutcome::ModuleGone(message)
6511 }
6512 }
6513 }
6514 Ok(probe) => {
6515 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
6516 {
6517 Ok(response) => ModuleControlRpcOutcome::Response(response),
6518 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6519 "malformed {} response body: {err}",
6520 probe.op
6521 )),
6522 };
6523 let completion = self
6524 .forwarding
6525 .complete_module_control_rpc(
6526 connection_id,
6527 frame.header.corr,
6528 Some(&probe.op),
6529 outcome,
6530 )
6531 .map_err(RouterError::Forwarding)?;
6532 if !self.observe_module_control_completion(completion) {
6533 debug!(
6534 connection_id = connection_id.get(),
6535 corr = frame.header.corr,
6536 op = %probe.op,
6537 "dropping late or unknown module-control RPC response"
6538 );
6539 }
6540 return Ok(Vec::new());
6541 }
6542 Err(err) => {
6543 if let Some(expected_op) = self
6544 .forwarding
6545 .pending_module_control_op(connection_id, frame.header.corr)
6546 .map_err(RouterError::Forwarding)?
6547 {
6548 let completion = self
6549 .forwarding
6550 .complete_module_control_rpc(
6551 connection_id,
6552 frame.header.corr,
6553 None,
6554 ModuleControlRpcOutcome::MalformedResponse(format!(
6555 "malformed {expected_op} response body: {err}"
6556 )),
6557 )
6558 .map_err(RouterError::Forwarding)?;
6559 if !self.observe_module_control_completion(completion) {
6560 debug!(
6561 connection_id = connection_id.get(),
6562 corr = frame.header.corr,
6563 "dropping late malformed module-control RPC response"
6564 );
6565 }
6566 return Ok(Vec::new());
6567 }
6568 let message = format!("malformed route.bind response body: {err}");
6569 secondary_error = Some(control_error_frame(
6570 &frame,
6571 "invalid_control_body",
6572 message.clone(),
6573 )?);
6574 RouteBindRelayOutcome::ModuleGone(message)
6575 }
6576 },
6577 FrameType::Error => {
6578 if self
6579 .forwarding
6580 .pending_module_control_op(connection_id, frame.header.corr)
6581 .map_err(RouterError::Forwarding)?
6582 .is_some()
6583 {
6584 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
6585 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
6586 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
6587 "malformed module-control ERROR body: {err}"
6588 )),
6589 };
6590 let completion = self
6591 .forwarding
6592 .complete_module_control_rpc(
6593 connection_id,
6594 frame.header.corr,
6595 None,
6596 outcome,
6597 )
6598 .map_err(RouterError::Forwarding)?;
6599 if !self.observe_module_control_completion(completion) {
6600 debug!(
6601 connection_id = connection_id.get(),
6602 corr = frame.header.corr,
6603 "dropping late or unknown module-control RPC error"
6604 );
6605 }
6606 return Ok(Vec::new());
6607 }
6608 match serde_json::from_slice::<ErrorBody>(&frame.body) {
6609 Ok(body) => RouteBindRelayOutcome::Rejected(body),
6610 Err(err) => {
6611 let message = format!("malformed route.bind ERROR body: {err}");
6612 secondary_error = Some(control_error_frame(
6613 &frame,
6614 "invalid_control_body",
6615 message.clone(),
6616 )?);
6617 RouteBindRelayOutcome::ModuleGone(message)
6618 }
6619 }
6620 }
6621 ty => {
6622 return Ok(vec![control_error_frame(
6623 &frame,
6624 "unsupported_control_frame",
6625 format!("unsupported module channel-0 frame {ty:?}"),
6626 )?])
6627 }
6628 };
6629
6630 let settled =
6631 self.forwarding
6632 .complete_pending_relay(connection_id, frame.header.corr, outcome);
6633 let completion = match settled {
6634 Ok(completion) => completion,
6635 Err(err) => {
6636 self.refuse_to_end_module_connection_for_a_client(
6637 connection_id,
6638 frame.header.corr,
6639 err,
6640 )?;
6641 return Ok(secondary_error.into_iter().collect());
6642 }
6643 };
6644 if let Some(target) = completion.abandoned.as_ref() {
6645 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
6646 }
6647 if !completion.settled {
6648 debug!(
6649 connection_id = connection_id.get(),
6650 corr = frame.header.corr,
6651 frame_type = ?frame.header.ty,
6652 "dropping late or unknown route.bind relay response"
6653 );
6654 }
6655 Ok(secondary_error.into_iter().collect())
6656 }
6657
6658 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
6659 debug!(connection_id = connection_id.get(), "handling GOODBYE");
6660 self.cleanup_connection_with_end_reason(
6664 connection_id,
6665 RegistrationEndReason::ExplicitGoodbye,
6666 )
6667 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
6668 Ok(Vec::new())
6669 }
6670}
6671
6672impl Default for ControlHandler {
6673 fn default() -> Self {
6674 Self::new(Arc::new(Registry::default()))
6675 }
6676}
6677
6678impl crate::supervise::SwapPromotionObserver for ControlHandler {
6679 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
6680 self.apply_registration_capabilities(registration);
6681 }
6682}
6683
6684fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
6685 CapabilityRequirementStatus {
6686 consumer: status.consumer,
6687 capability: status.capability,
6688 need: match status.need {
6689 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
6690 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
6691 },
6692 verdict: status.verdict.as_str().to_string(),
6693 episode_seq: status.episode_seq,
6694 config_satisfiable: status.config_satisfiable,
6695 runtime_available: status.runtime_available,
6696 detail: status.detail,
6697 }
6698}
6699
6700fn append_capability_problem_detail(
6701 detail: Option<String>,
6702 capability_detail: Option<String>,
6703) -> Option<String> {
6704 match (detail, capability_detail) {
6705 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
6706 (Some(detail), None) => Some(detail),
6707 (None, Some(capability_detail)) => Some(capability_detail),
6708 (None, None) => None,
6709 }
6710}
6711
6712fn subc_ops() -> Vec<String> {
6713 SUBC_CONTROL_OPS
6714 .iter()
6715 .map(|op| (*op).to_string())
6716 .collect()
6717}
6718
6719fn module_subc_ops() -> Vec<String> {
6720 SUBC_CONTROL_OPS
6721 .iter()
6722 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
6723 .map(|op| (*op).to_string())
6724 .collect()
6725}
6726
6727#[cfg(test)]
6728fn module_baseline_control_ops() -> Vec<String> {
6729 MODULE_BASELINE_CONTROL_OPS
6730 .iter()
6731 .map(|op| (*op).to_string())
6732 .collect()
6733}
6734
6735fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
6736 let mut seen = HashSet::new();
6737 let mut effective = Vec::new();
6738 for op in MODULE_BASELINE_CONTROL_OPS {
6739 if seen.insert((*op).to_string()) {
6740 effective.push((*op).to_string());
6741 }
6742 }
6743 for op in declared.unwrap_or_default() {
6744 if seen.insert(op.clone()) {
6745 effective.push(op);
6746 }
6747 }
6748 effective
6749}
6750
6751fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
6752 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
6753}
6754
6755fn target_module_id(target: &RouteTarget) -> &str {
6756 match target {
6757 RouteTarget::ToolProvider { module_id }
6758 | RouteTarget::ManagementSurface { module_id }
6759 | RouteTarget::InternalService { module_id, .. } => module_id,
6760 }
6761}
6762
6763fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
6764 roles.iter().any(|role| match (target, role) {
6765 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
6766 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
6767 (
6768 RouteTarget::InternalService { service_id, .. },
6769 ProviderRole::InternalService {
6770 service_id: provided,
6771 ..
6772 },
6773 ) => service_id == provided,
6774 _ => false,
6775 })
6776}
6777
6778fn is_routable_role(role: &ProviderRole) -> bool {
6779 matches!(
6780 role,
6781 ProviderRole::ToolProvider { .. }
6782 | ProviderRole::ManagementSurface { .. }
6783 | ProviderRole::InternalService { .. }
6784 )
6785}
6786
6787#[derive(Debug, Clone, Copy, PartialEq, Eq)]
6788enum ControlRequestBodyError {
6789 UnknownOp,
6790 InvalidBody,
6791}
6792
6793#[derive(Debug, Deserialize)]
6794struct ControlOpProbe {
6795 op: String,
6796}
6797
6798const MODULE_PUSH_OPS: &[&str] = &["route.status"];
6801
6802fn is_known_module_push_op(body: &[u8]) -> bool {
6803 serde_json::from_slice::<ControlOpProbe>(body)
6804 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
6805 .unwrap_or(false)
6806}
6807
6808fn is_known_module_request_op(body: &[u8]) -> bool {
6809 serde_json::from_slice::<ControlOpProbe>(body)
6810 .map(|probe| is_module_to_subc_op(&probe.op))
6811 .unwrap_or(false)
6812}
6813
6814fn is_module_to_subc_op(op: &str) -> bool {
6815 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
6816}
6817
6818fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
6819 debug!(
6820 op = %op,
6821 connection_id = connection_id.get(),
6822 corr,
6823 "control dispatch"
6824 );
6825}
6826
6827fn log_slow_control_dispatch(
6828 dispatch_started_at: Option<StdInstant>,
6829 op: &'static str,
6830 connection_id: ConnectionId,
6831 corr: u64,
6832 route_open_timings: Option<&RouteOpenTimings>,
6833) {
6834 let Some(dispatch_started_at) = dispatch_started_at else {
6835 return;
6836 };
6837 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
6839 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
6840 if let Some(timings) = route_open_timings {
6841 warn!(
6842 op = %op,
6843 connection_id = connection_id.get(),
6844 corr,
6845 elapsed_ms = elapsed.as_millis() as u64,
6846 module_id = ?timings.module_id,
6847 principal = %timings.principal,
6848 admission_ms = timings.admission.as_millis() as u64,
6849 project_root_ms = timings.project_root.as_millis() as u64,
6850 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
6851 bind_send_ms = timings.bind_send.as_millis() as u64,
6852 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
6853 "slow control dispatch"
6854 );
6855 return;
6856 }
6857 warn!(
6858 op = %op,
6859 connection_id = connection_id.get(),
6860 corr,
6861 elapsed_ms = elapsed.as_millis() as u64,
6862 "slow control dispatch"
6863 );
6864 }
6865}
6866
6867#[cfg(test)]
6869type TestCallerLookup =
6870 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
6871
6872fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
6874 match request {
6875 ClientControlRequest::SupervisorRestart { module_id, .. } => {
6876 Some(PrivilegedOperation::Restart { module_id })
6877 }
6878 ClientControlRequest::SupervisorSwap { module_id, .. } => {
6879 Some(PrivilegedOperation::Swap { module_id })
6880 }
6881 ClientControlRequest::SupervisorReload { module_id } => {
6882 Some(PrivilegedOperation::Reload { module_id })
6883 }
6884 ClientControlRequest::SupervisorRescan { preview } => {
6885 if *preview {
6886 None
6887 } else {
6888 Some(PrivilegedOperation::Rescan)
6889 }
6890 }
6891 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
6892 Some(PrivilegedOperation::SetEnabled {
6893 module_id,
6894 enabled: *enabled,
6895 })
6896 }
6897 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
6898 Some(PrivilegedOperation::ReleaseReserved { module_id })
6899 }
6900 ClientControlRequest::ServerDescribe {}
6901 | ClientControlRequest::CatalogList { .. }
6902 | ClientControlRequest::RouteOpen { .. }
6903 | ClientControlRequest::RoutePoll { .. }
6904 | ClientControlRequest::SupervisorList {}
6905 | ClientControlRequest::SupervisorHealth {}
6906 | ClientControlRequest::SupervisorHealthProbe { .. }
6907 | ClientControlRequest::SupervisorRoutes { .. }
6908 | ClientControlRequest::SupervisorProvenance { .. }
6909 | ClientControlRequest::SupervisorStderrTail { .. }
6910 | ClientControlRequest::SupervisorTerminals { .. }
6911 | ClientControlRequest::SupervisorSpawnSnapshot {}
6912 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
6913 }
6914}
6915
6916fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
6917 match request {
6918 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
6919 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
6920 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
6921 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
6922 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
6923 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
6924 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
6925 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
6926 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
6927 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
6928 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
6929 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
6930 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
6931 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
6932 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
6933 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
6934 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
6935 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
6936 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
6937 }
6938}
6939
6940fn principal_label(principal: &Principal) -> String {
6941 match principal {
6942 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
6943 Principal::Direct => "direct".to_string(),
6944 other => format!("{other:?}"),
6945 }
6946}
6947
6948fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
6949 match request {
6950 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
6951 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
6952 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
6953 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
6954 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
6955 }
6956}
6957
6958fn parse_client_control_request(
6959 body: &[u8],
6960) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
6961 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
6962 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6963 Ok(probe) if SUBC_CONTROL_OPS.contains(&probe.op.as_str()) => {
6964 ControlRequestBodyError::InvalidBody
6965 }
6966 Ok(_) => ControlRequestBodyError::UnknownOp,
6967 Err(_) => ControlRequestBodyError::InvalidBody,
6968 };
6969 (err, classification)
6970 })
6971}
6972
6973fn parse_module_control_request_from_module(
6974 body: &[u8],
6975) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
6976 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
6977 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
6978 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
6979 Ok(_) => ControlRequestBodyError::UnknownOp,
6980 Err(_) => ControlRequestBodyError::InvalidBody,
6981 };
6982 (err, classification)
6983 })
6984}
6985
6986#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
6987enum ProviderRoleKind {
6988 ToolProvider,
6989 PipelineStage,
6990 ManagementSurface,
6991 InternalService,
6992}
6993
6994fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
6995 match role {
6996 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
6997 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
6998 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
6999 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7000 }
7001}
7002
7003fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7004 roles.iter().map(provider_role_kind).collect()
7005}
7006
7007const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7010
7011#[derive(Debug, Default, PartialEq, Eq)]
7013struct ScopeOutcomeCounts {
7014 created: usize,
7015 replaced: usize,
7016 updated: usize,
7017 unchanged: usize,
7018 refused: usize,
7019}
7020
7021impl ScopeOutcomeCounts {
7022 fn of(results: &[ScopeRecordResult]) -> Self {
7023 let mut counts = Self::default();
7024 for result in results {
7025 let slot = match result.outcome {
7026 ScopeRecordOutcome::Created => &mut counts.created,
7027 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7028 ScopeRecordOutcome::Updated => &mut counts.updated,
7029 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7030 ScopeRecordOutcome::Refused => &mut counts.refused,
7031 };
7032 *slot += 1;
7033 }
7034 counts
7035 }
7036}
7037
7038#[cfg(test)]
7039mod scope_outcome_count_tests {
7040 use super::*;
7041
7042 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7043 ScopeRecordResult {
7044 scope_ref: "r".to_string(),
7045 scope_epoch: 1,
7046 outcome,
7047 code: None,
7048 message: None,
7049 version: None,
7050 parent_state: None,
7051 }
7052 }
7053
7054 #[test]
7057 fn every_outcome_is_counted_in_its_own_field() {
7058 let results = [
7059 result(ScopeRecordOutcome::Created),
7060 result(ScopeRecordOutcome::Created),
7061 result(ScopeRecordOutcome::Replaced),
7062 result(ScopeRecordOutcome::Updated),
7063 result(ScopeRecordOutcome::Unchanged),
7064 result(ScopeRecordOutcome::Refused),
7065 result(ScopeRecordOutcome::Refused),
7066 result(ScopeRecordOutcome::Refused),
7067 ];
7068 assert_eq!(
7069 ScopeOutcomeCounts::of(&results),
7070 ScopeOutcomeCounts {
7071 created: 2,
7072 replaced: 1,
7073 updated: 1,
7074 unchanged: 1,
7075 refused: 3,
7076 }
7077 );
7078 }
7079}
7080
7081fn capability_census_trigger(
7085 old: Option<&CapabilityDeclarations>,
7086 new: Option<&CapabilityDeclarations>,
7087) -> bool {
7088 let old_provides = old
7089 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7090 .unwrap_or_default();
7091 let old_denies = old
7092 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7093 .unwrap_or_default();
7094 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7095 provides: Vec::new(),
7096 requires: Vec::new(),
7097 must_never_reach: Vec::new(),
7098 });
7099
7100 new.provides
7101 .iter()
7102 .any(|capability| !old_provides.contains(capability))
7103 || new
7104 .must_never_reach
7105 .iter()
7106 .any(|capability| !old_denies.contains(capability))
7107}
7108
7109fn denied_capability<'a>(
7112 opening_manifest: &'a ModuleManifest,
7113 target_manifest: &ModuleManifest,
7114) -> Option<&'a str> {
7115 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7116 let target_capabilities = target_manifest.capabilities.as_ref()?;
7117 opening_capabilities
7118 .must_never_reach
7119 .iter()
7120 .find(|denied| {
7121 target_capabilities
7122 .provides
7123 .iter()
7124 .any(|provided| provided == *denied)
7125 })
7126 .map(String::as_str)
7127}
7128
7129fn catalog_update_frozen_field_message(
7130 registered: &ModuleManifest,
7131 provides: &[ProviderRole],
7132) -> Option<String> {
7133 let old_has_provides = !registered.provides.is_empty();
7134 let new_has_provides = !provides.is_empty();
7135 if old_has_provides != new_has_provides {
7136 return Some(format!(
7137 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7138 registered.module_id
7139 ));
7140 }
7141
7142 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7143 return Some(format!(
7144 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7145 registered.module_id
7146 ));
7147 }
7148
7149 let registered_concurrency = manifest_concurrency(registered);
7150 let mut candidate = registered.clone();
7151 candidate.provides = provides.to_vec();
7152 let candidate_concurrency = manifest_concurrency(&candidate);
7153 if candidate_concurrency != registered_concurrency {
7154 return Some(format!(
7155 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7156 registered.module_id, registered_concurrency, candidate_concurrency
7157 ));
7158 }
7159
7160 None
7163}
7164
7165fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7166 manifest.provides.iter().any(is_routable_role)
7167}
7168
7169fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7175 manifest
7176 .provides
7177 .iter()
7178 .find_map(|provider| match provider {
7179 ProviderRole::ToolProvider { concurrency, .. }
7180 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7181 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7182 })
7183 .unwrap_or(Concurrency::ModuleManaged)
7184}
7185
7186fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7193 let has_management_surface = manifest
7194 .provides
7195 .iter()
7196 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7197 if !has_management_surface {
7198 return false;
7199 }
7200 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7201 return false;
7202 };
7203 let Some(provides) = raw
7204 .get("manifest")
7205 .and_then(|manifest| manifest.get("provides"))
7206 .and_then(serde_json::Value::as_array)
7207 else {
7208 return false;
7209 };
7210 provides.iter().any(|role| {
7215 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
7216 && role.get("concurrency").is_none()
7217 })
7218}
7219
7220fn negotiate_version(peer_version: u8) -> Result<u8, String> {
7221 if peer_version != PROTOCOL_VERSION {
7222 return Err(format!(
7223 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
7224 ));
7225 }
7226 Ok(PROTOCOL_VERSION)
7227}
7228
7229fn pong(frame: &Frame) -> Result<Frame, RouterError> {
7230 Frame::build_with_version(
7231 response_version(frame),
7232 FrameType::Pong,
7233 frame.header.flags,
7234 0,
7235 0,
7236 frame.header.corr,
7237 Vec::new(),
7238 )
7239 .map_err(RouterError::FrameBuild)
7240}
7241
7242pub(crate) fn control_error_frame(
7243 frame: &Frame,
7244 code: &'static str,
7245 message: impl Into<String>,
7246) -> Result<Frame, RouterError> {
7247 control_error_body_frame(
7248 frame,
7249 ErrorBody {
7250 code: code.to_string(),
7251 message: message.into(),
7252 detail: None,
7253 },
7254 )
7255}
7256
7257fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
7258 let body = serde_json::to_vec(&error).map_err(|err| {
7259 RouterError::backend(
7260 0,
7261 frame.header.corr,
7262 format!("failed to encode control ERROR: {err}"),
7263 )
7264 })?;
7265
7266 Frame::build_with_version(
7267 response_version(frame),
7268 FrameType::Error,
7269 control_flags(),
7270 0,
7271 0,
7272 frame.header.corr,
7273 body,
7274 )
7275 .map_err(RouterError::FrameBuild)
7276}
7277
7278fn control_response_body_frame<T: Serialize>(
7279 frame: &Frame,
7280 reply: &T,
7281 label: &'static str,
7282) -> Result<Frame, RouterError> {
7283 let body = serde_json::to_vec(reply).map_err(|err| {
7284 RouterError::backend(
7285 0,
7286 frame.header.corr,
7287 format!("failed to encode {label}: {err}"),
7288 )
7289 })?;
7290
7291 Frame::build_with_version(
7292 response_version(frame),
7293 FrameType::Response,
7294 control_flags(),
7295 0,
7296 0,
7297 frame.header.corr,
7298 body,
7299 )
7300 .map_err(RouterError::FrameBuild)
7301}
7302
7303fn forwarding_error_code(err: &ForwardingError) -> &'static str {
7333 match err {
7334 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
7335 ForwardingError::NoModuleConnection => "target_unavailable",
7336 ForwardingError::ModuleReloading { .. } => "module_reloading",
7337 ForwardingError::ClientRouteChannelExhausted { .. }
7338 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
7339 ForwardingError::StaleModuleEndpoint
7340 | ForwardingError::UnknownReservation { .. }
7341 | ForwardingError::ConnectionClosing { .. }
7342 | ForwardingError::ClientEgressClosed { .. }
7343 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
7344 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
7347 ForwardingError::RelayCorrelationExhausted
7348 | ForwardingError::RouteOpenBuild(_)
7349 | ForwardingError::Poisoned => "forwarding_error",
7350 }
7351}
7352
7353fn response_version(frame: &Frame) -> u8 {
7354 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
7355 frame.header.ver
7356 } else {
7357 PROTOCOL_VERSION
7358 }
7359}
7360
7361fn control_flags() -> Flags {
7362 Flags::new(false, Priority::Passive, false)
7363}
7364
7365fn send_goodbye_target_best_effort(
7370 counters: &DaemonCounters,
7371 target: &GoodbyeTarget,
7372 context: &'static str,
7373) {
7374 let Ok(frame) = Frame::build_with_version(
7375 target.negotiated_ver,
7376 FrameType::Goodbye,
7377 control_flags(),
7378 target.channel,
7379 target.epoch,
7380 0,
7381 Vec::new(),
7382 ) else {
7383 return;
7384 };
7385 crate::forwarding::send_module_route_goodbye(
7386 counters,
7387 &target.sink,
7388 frame,
7389 target.module_id.as_deref(),
7390 context,
7391 );
7392}
7393
7394pub(crate) fn send_route_control_pushes(
7395 forwarding: &ForwardingTable,
7396 routes: Vec<EndpointRoute>,
7397 push: ClientControlPush,
7398) {
7399 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
7400 for route in routes {
7401 let target = route.goodbye_target;
7402 if let Some((existing, channels)) = targets
7403 .iter_mut()
7404 .find(|(existing, _)| existing.connection_id == target.connection_id)
7405 {
7406 debug_assert_eq!(
7407 existing.negotiated_ver, target.negotiated_ver,
7408 "one connection cannot negotiate multiple frame versions"
7409 );
7410 if !channels.contains(&target.channel) {
7411 channels.push(target.channel);
7412 }
7413 continue;
7414 }
7415 let channel = target.channel;
7416 targets.push((target, vec![channel]));
7417 }
7418 for (target, mut channels) in targets {
7419 channels.sort_unstable();
7420 let mut push = push.clone();
7421 match &mut push {
7422 ClientControlPush::RouteClosing {
7423 channels: covered, ..
7424 }
7425 | ClientControlPush::RouteClosed {
7426 channels: covered, ..
7427 } => *covered = channels,
7428 }
7429 let body = match serde_json::to_vec(&push) {
7430 Ok(body) => body,
7431 Err(err) => {
7432 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
7433 continue;
7434 }
7435 };
7436 let frame = match Frame::build_with_version(
7437 target.negotiated_ver,
7438 FrameType::Push,
7439 control_flags(),
7440 0,
7441 0,
7442 0,
7443 body.clone(),
7444 ) {
7445 Ok(frame) => frame,
7446 Err(err) => {
7447 warn!(
7448 route_channel = target.channel,
7449 error = %err,
7450 "failed to build route lifecycle control PUSH frame"
7451 );
7452 continue;
7453 }
7454 };
7455 if let Err(err) = target.sink.try_send(frame) {
7456 if target.close_on_delivery_failure() {
7457 warn!(
7458 target_connection_id = target.connection_id.get(),
7459 route_channel = target.channel,
7460 error = %err,
7461 "route lifecycle control PUSH was not delivered to client; closing target connection"
7462 );
7463 let _ = forwarding.escalate_client_delivery_failure(
7464 target.connection_id,
7465 target.channel,
7466 target.epoch,
7467 CloseReason::new(
7468 "route_lifecycle_push_delivery_failed",
7469 format!(
7470 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
7471 target.channel
7472 ),
7473 ),
7474 crate::forwarding::UndeliveredFrame {
7475 module_id: target.module_id.as_deref(),
7476 sink: &target.sink,
7477 },
7478 );
7479 }
7480 }
7481 }
7482}
7483
7484#[cfg(test)]
7485mod tests {
7486 mod activity_read {
7487 include!("activity_read_tests.rs");
7488 }
7489
7490 mod lifecycle_activity {
7491 include!("control_lifecycle_tests.rs");
7492 }
7493 #[cfg(unix)]
7494 #[tokio::test]
7495 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
7496 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
7497 let path = dir.join("subc.jsonc");
7498 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
7499 "program":"/bin/sleep","args":["60"],"protocol":"none",
7500 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
7501 }}}).to_string()).unwrap();
7502 let mut configured = crate::daemon_config::load(&path)
7503 .unwrap()
7504 .unwrap()
7505 .modules
7506 .pop()
7507 .unwrap();
7508 let registry = std::sync::Arc::new(crate::Registry::default());
7509 let handle = crate::SupervisorHandle::new();
7510 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
7511 .with_handle(handle.clone());
7512 let module = supervisor
7513 .supervise_configured_with_health(
7514 configured.module_spec(),
7515 true,
7516 configured.health.clone(),
7517 None,
7518 configured.restart,
7519 )
7520 .unwrap();
7521 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
7522 let before = module.status().unwrap().pid;
7523 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
7524 configured.health.cadence = std::time::Duration::from_secs(3600);
7525 let health_only = handler
7526 .reconcile_supervised_modules(
7527 &supervisor,
7528 vec![configured.clone()],
7529 false,
7530 Principal::Direct,
7531 )
7532 .await
7533 .unwrap();
7534 assert!(
7535 health_only.changed_pending_reload.is_empty(),
7536 "health policy is already applied live"
7537 );
7538 assert_eq!(module.status().unwrap().pid, before);
7539 assert_eq!(
7540 module.configuration().unwrap().1.http,
7541 configured.health.http
7542 );
7543 configured.args = vec!["61".into()];
7544 let launch = handler
7545 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
7546 .await
7547 .unwrap();
7548 assert_eq!(launch.changed_pending_reload, ["stock"]);
7549 assert_eq!(
7550 module.status().unwrap().pid,
7551 before,
7552 "a launch edit is stored until reload"
7553 );
7554 module.drain().await.unwrap();
7555 }
7556 use cortexkit_test_support::ScratchDir;
7557 use std::{
7558 collections::BTreeMap,
7559 fmt,
7560 path::PathBuf,
7561 sync::{Arc, Mutex},
7562 time::Duration,
7563 };
7564
7565 use serde_json::{json, Value};
7566 use subc_protocol::{
7567 manifest::{
7568 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
7569 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
7570 },
7571 session::HealthStatus,
7572 FrameType,
7573 };
7574
7575 use super::*;
7576 use crate::{
7577 forwarding::{DataRoute, DataRouteState},
7578 registry::ChannelState,
7579 router::FrameSink,
7580 stderr_tail::DEFAULT_MAX_LINE_BYTES,
7581 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
7582 RouteCtx, Router,
7583 };
7584 use tokio::{
7585 sync::mpsc,
7586 time::{sleep, Instant},
7587 };
7588 use tracing::{
7589 field::{Field, Visit},
7590 Event, Subscriber,
7591 };
7592 use tracing_subscriber::{layer::Context, prelude::*, Layer};
7593
7594 fn fake_aft_stub_path() -> PathBuf {
7613 let mut path = std::env::current_exe().expect("current_exe available in tests");
7614 path.pop(); path.pop(); path.push(if cfg!(windows) {
7617 "fake-aft-stub.exe"
7618 } else {
7619 "fake-aft-stub"
7620 });
7621 assert!(
7622 path.exists(),
7623 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
7624 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
7625 path.display()
7626 );
7627 path
7628 }
7629
7630 fn client_retries(code: &str) -> bool {
7635 subc_protocol::error_codes::is_retryable_route_open(code)
7636 }
7637
7638 #[test]
7649 fn retryability_of_forwarding_codes_matches_the_failure() {
7650 let transient = [
7653 ForwardingError::NoModuleConnection,
7654 ForwardingError::ModuleReloading {
7655 module_id: "m".into(),
7656 },
7657 ForwardingError::StaleModuleEndpoint,
7658 ForwardingError::UnknownReservation {
7659 client_channel: 1,
7660 module_channel: 1,
7661 },
7662 ForwardingError::ConnectionClosing {
7663 connection_id: ConnectionId::new(1),
7664 },
7665 ForwardingError::ClientEgressClosed {
7666 connection_id: ConnectionId::new(1),
7667 },
7668 ForwardingError::ModuleEgressUnavailable {
7669 connection_id: ConnectionId::new(1),
7670 },
7671 ];
7672 for err in transient {
7673 let code = forwarding_error_code(&err);
7674 assert!(
7675 client_retries(code),
7676 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
7677 );
7678 }
7679
7680 let permanent = [
7685 ForwardingError::ConnectionRoleConflict {
7686 connection_id: ConnectionId::new(1),
7687 },
7688 ForwardingError::ClientRouteChannelExhausted {
7689 connection_id: ConnectionId::new(1),
7690 },
7691 ForwardingError::ModuleRouteChannelExhausted {
7692 endpoint: ModuleEndpointId {
7693 connection_id: ConnectionId::new(1),
7694 generation: 1,
7695 },
7696 },
7697 ForwardingError::RelayCorrelationExhausted,
7698 ForwardingError::RouteOpenBuild("x".into()),
7699 ForwardingError::Poisoned,
7700 ];
7701 for err in permanent {
7702 let code = forwarding_error_code(&err);
7703 assert!(
7704 !client_retries(code),
7705 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
7706 );
7707 }
7708 }
7709
7710 #[tokio::test]
7738 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
7739 let handler = ControlHandler::default();
7740 let frame =
7741 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
7742
7743 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
7747 assert_eq!(
7748 stamped,
7749 Principal::Direct,
7750 "a caller that proved nothing must not be stamped as a supervised module"
7751 );
7752
7753 let forged = handler
7758 .route_open_principal(
7759 &frame,
7760 Some(ConsumerIdentity {
7761 module_id: "aft".to_string(),
7762 launch_nonce: "not-a-real-nonce".to_string(),
7763 }),
7764 )
7765 .unwrap();
7766 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
7767 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
7768 }
7769
7770 #[test]
7787 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
7788 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
7789 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
7790 let ClientControlRequest::RouteOpen {
7791 consumer_identity, ..
7792 } = parsed
7793 else {
7794 panic!("route.open body must parse as RouteOpen");
7795 };
7796 assert_eq!(
7797 consumer_identity,
7798 Some(ConsumerIdentity {
7799 module_id: "aft".to_string(),
7800 launch_nonce: "n".to_string(),
7801 }),
7802 "the wire field name must reach the value route_open_principal reads"
7803 );
7804 }
7805
7806 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
7807 ModuleManifest::builder(module_id, "0.1.0")
7808 .protocol_ver(protocol_ver)
7809 .provides(vec![ProviderRole::ToolProvider {
7810 tools: vec![Tool {
7811 name: "read".to_string(),
7812 description: None,
7813 execution_mode: ExecutionMode::Pure,
7814 schema: json!({"type": "object"}),
7815 }],
7816 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
7817 concurrency: Concurrency::ModuleManaged,
7818 emits_push: true,
7819 sub_supervises: true,
7820 }])
7821 .build()
7822 }
7823
7824 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
7825 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
7826 }
7827
7828 fn hello_frame_with_control_ops(
7829 module_id: &str,
7830 protocol_ver: u8,
7831 corr: u64,
7832 control_ops: Option<Vec<String>>,
7833 ) -> Frame {
7834 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
7835 }
7836
7837 fn hello_frame_with_nonce(
7838 module_id: &str,
7839 protocol_ver: u8,
7840 corr: u64,
7841 launch_nonce: Option<&str>,
7842 ) -> Frame {
7843 hello_frame_full(
7844 module_id,
7845 protocol_ver,
7846 corr,
7847 None,
7848 launch_nonce.map(ToOwned::to_owned),
7849 )
7850 }
7851
7852 fn hello_frame_full(
7853 module_id: &str,
7854 protocol_ver: u8,
7855 corr: u64,
7856 control_ops: Option<Vec<String>>,
7857 launch_nonce: Option<String>,
7858 ) -> Frame {
7859 let body = serde_json::to_vec(&ModuleHelloBody {
7860 manifest: manifest(module_id, protocol_ver),
7861 protocol_ver,
7862 control_ops,
7863 launch_nonce,
7864 })
7865 .unwrap();
7866 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7867 }
7868
7869 fn non_routable_hello_frame_with_control_ops(
7870 module_id: &str,
7871 corr: u64,
7872 control_ops: Option<Vec<String>>,
7873 ) -> Frame {
7874 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
7875 manifest.provides.clear();
7876 let body = serde_json::to_vec(&ModuleHelloBody {
7877 manifest,
7878 protocol_ver: PROTOCOL_VERSION,
7879 control_ops,
7880 launch_nonce: None,
7881 })
7882 .unwrap();
7883 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
7884 }
7885
7886 fn capability_grammar_hello_frame(
7887 capabilities: Value,
7888 runtime_computed: Option<Value>,
7889 corr: u64,
7890 ) -> Frame {
7891 let mut body = serde_json::to_value(ModuleHelloBody {
7892 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
7893 protocol_ver: PROTOCOL_VERSION,
7894 control_ops: None,
7895 launch_nonce: None,
7896 })
7897 .expect("HELLO body serializes");
7898 body["manifest"]["capabilities"] = capabilities;
7899 if let Some(runtime_computed) = runtime_computed {
7900 body["runtime_computed"] = runtime_computed;
7901 }
7902 Frame::build(
7903 FrameType::Hello,
7904 control_flags(),
7905 0,
7906 0,
7907 corr,
7908 serde_json::to_vec(&body).expect("HELLO body reserializes"),
7909 )
7910 .expect("HELLO frame builds")
7911 }
7912
7913 fn channel_request(channel: u16, corr: u64) -> Frame {
7914 Frame::build(
7915 FrameType::Request,
7916 Flags::new(true, Priority::Interactive, false),
7917 channel,
7918 0,
7919 corr,
7920 b"opaque".to_vec(),
7921 )
7922 .unwrap()
7923 }
7924
7925 fn route_ctx(
7926 connection_id: ConnectionId,
7927 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
7928 let (tx, rx) = mpsc::channel(8);
7929 (
7930 RouteCtx {
7931 connection_id,
7932 egress: FrameSink::new(tx),
7933 },
7934 rx,
7935 )
7936 }
7937
7938 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
7939 serde_json::from_slice(&frame.body).unwrap()
7940 }
7941
7942 async fn hello_via_sink(
7947 handler: &ControlHandler,
7948 ctx: &RouteCtx,
7949 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
7950 hello: Frame,
7951 ) -> Frame {
7952 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
7953 assert!(
7954 replies.is_empty(),
7955 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
7956 );
7957 let ack = rx
7958 .try_recv()
7959 .expect("HELLO_ACK is queued on the module sink")
7960 .frame;
7961 assert_eq!(ack.header.ty, FrameType::HelloAck);
7962 ack
7963 }
7964
7965 fn parse_error(frame: &Frame) -> Value {
7966 serde_json::from_slice(&frame.body).unwrap()
7967 }
7968
7969 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
7970 serde_json::from_slice(&frame.body).unwrap()
7971 }
7972
7973 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
7974 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
7975 route_channel,
7976 route_epoch: 0,
7977 kind,
7978 })
7979 .unwrap();
7980 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7981 }
7982
7983 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
7984 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
7985 module_id: module_id.to_string(),
7986 })
7987 .unwrap();
7988 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
7989 }
7990
7991 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
7992 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
7993 }
7994
7995 fn route_open_frame_with_consumer_capabilities(
7996 corr: u64,
7997 module_id: &str,
7998 project_root: ScratchDir,
7999 consumer_capabilities: Option<Vec<String>>,
8000 ) -> Frame {
8001 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8002 target: RouteTarget::ToolProvider {
8003 module_id: module_id.to_string(),
8004 },
8005 identity: BindIdentity::new(
8006 project_root.path().to_path_buf(),
8007 "unit".to_string(),
8008 "session".to_string(),
8009 ),
8010 consumer_identity: None,
8011 consumer_capabilities,
8012 role_versions: None,
8013 admission_facts: None,
8014 scope: None,
8015 })
8016 .unwrap();
8017 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8018 }
8019
8020 fn route_open_frame_with_role_versions(
8021 corr: u64,
8022 module_id: &str,
8023 project_root: ScratchDir,
8024 role_versions: Option<BTreeMap<String, String>>,
8025 ) -> Frame {
8026 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8027 target: RouteTarget::ToolProvider {
8028 module_id: module_id.to_string(),
8029 },
8030 identity: BindIdentity::new(
8031 project_root.path().to_path_buf(),
8032 "unit".to_string(),
8033 format!("session-{corr}"),
8034 ),
8035 consumer_identity: None,
8036 consumer_capabilities: None,
8037 role_versions,
8038 admission_facts: None,
8039 scope: None,
8040 })
8041 .unwrap();
8042 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8043 }
8044
8045 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8046 entries
8047 .iter()
8048 .map(|(role, version)| (role.to_string(), version.to_string()))
8049 .collect()
8050 }
8051
8052 fn route_open_frame_with_admission_facts(
8053 corr: u64,
8054 module_id: &str,
8055 project_root: ScratchDir,
8056 consumer_identity: Option<subc_control::ConsumerIdentity>,
8057 facts: Option<Value>,
8058 ) -> Frame {
8059 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8060 target: RouteTarget::ToolProvider {
8061 module_id: module_id.to_string(),
8062 },
8063 identity: BindIdentity::new(
8064 project_root.path().to_path_buf(),
8065 "unit".to_string(),
8066 format!("session-{corr}"),
8067 ),
8068 consumer_identity,
8069 consumer_capabilities: None,
8070 role_versions: None,
8071 admission_facts: facts,
8072 scope: None,
8073 })
8074 .unwrap();
8075 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8076 }
8077
8078 #[derive(Clone, Default)]
8079 struct EventCapture {
8080 events: Arc<Mutex<Vec<CapturedEvent>>>,
8081 }
8082
8083 #[derive(Clone, Debug)]
8084 struct CapturedEvent {
8085 target: String,
8086 level: tracing::Level,
8087 fields: BTreeMap<String, String>,
8088 }
8089
8090 impl EventCapture {
8091 fn events(&self) -> Vec<CapturedEvent> {
8092 self.events.lock().unwrap().clone()
8093 }
8094 }
8095
8096 impl<S> Layer<S> for EventCapture
8097 where
8098 S: Subscriber,
8099 {
8100 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8101 let mut visitor = EventFieldVisitor::default();
8102 event.record(&mut visitor);
8103 self.events.lock().unwrap().push(CapturedEvent {
8104 target: event.metadata().target().to_string(),
8105 level: *event.metadata().level(),
8106 fields: visitor.fields,
8107 });
8108 }
8109 }
8110
8111 #[derive(Default)]
8112 struct EventFieldVisitor {
8113 fields: BTreeMap<String, String>,
8114 }
8115
8116 impl Visit for EventFieldVisitor {
8117 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8118 self.fields
8119 .insert(field.name().to_string(), format!("{value:?}"));
8120 }
8121 }
8122
8123 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8124 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8125 status,
8126 detail: Some("warming".to_string()),
8127 metrics: Some(json!({"queue_depth": 3})),
8128 })
8129 .unwrap();
8130 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8131 }
8132
8133 fn route_bind_ack(corr: u64) -> Frame {
8134 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8135 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8136 }
8137
8138 fn unique_project_root(label: &str) -> ScratchDir {
8139 ScratchDir::new(label)
8140 }
8141
8142 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8143 match parse_route_poll(frame) {
8144 ClientControlResponse::RoutePoll {
8145 status: None,
8146 live: Some(live),
8147 ..
8148 } => assert_eq!(live, expected_live),
8149 other => panic!("unexpected route.poll response: {other:?}"),
8150 }
8151 }
8152
8153 fn bind_liveness_route(
8154 registry: &Registry,
8155 forwarding: &ForwardingTable,
8156 module_id: &str,
8157 ) -> (RouteCtx, u16, u32) {
8158 let module_connection = ConnectionId::new(101);
8159 let client_connection = ConnectionId::new(202);
8160 let registration = registry
8161 .register_with_control_ops(
8162 manifest(module_id, PROTOCOL_VERSION),
8163 PROTOCOL_VERSION,
8164 module_connection,
8165 module_baseline_control_ops(),
8166 )
8167 .unwrap();
8168 let (module_tx, _module_rx) = mpsc::channel(8);
8169 let endpoint = forwarding
8170 .register_module_connection(
8171 module_connection,
8172 module_id.to_string(),
8173 PROTOCOL_VERSION,
8174 manifest_concurrency(®istration.manifest),
8175 FrameSink::new(module_tx),
8176 )
8177 .unwrap();
8178 let (client_ctx, _client_rx) = route_ctx(client_connection);
8179 let pending = forwarding
8180 .begin_route_bind_relay_for_test(
8181 client_connection,
8182 client_ctx.egress.clone(),
8183 1,
8184 module_id,
8185 )
8186 .unwrap();
8187 assert_eq!(pending.endpoint, endpoint);
8188 let route_channel = pending.client_channel;
8189 let route_epoch = pending.client_epoch;
8190 forwarding
8191 .complete_pending_relay(
8192 module_connection,
8193 pending.corr,
8194 RouteBindRelayOutcome::Accepted,
8195 )
8196 .unwrap();
8197 (client_ctx, route_channel, route_epoch)
8198 }
8199
8200 struct FakeProcessLiveness {
8201 live: Option<bool>,
8202 }
8203
8204 impl ModuleProcessLiveness for FakeProcessLiveness {
8205 fn process_live(&self, _module_id: &str) -> Option<bool> {
8206 self.live
8207 }
8208 }
8209
8210 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8211 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
8212 {
8213 let registry = Arc::new(Registry::default());
8214 let supervisor_handle = SupervisorHandle::new();
8215 let supervisor = Supervisor::new_for_test(
8216 Arc::clone(®istry),
8217 RestartPolicy::new(1, Duration::from_millis(10)),
8218 )
8219 .with_handle(supervisor_handle.clone());
8220 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
8221 let module = supervisor
8222 .spawn(ModuleSpec {
8223 module_id: "stderr-tail-wire".to_string(),
8224 program: fake_aft_stub_path(),
8225 args: Vec::new(),
8226 env: vec![
8227 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
8228 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
8229 ],
8230 reserved: false,
8231 reserved_prefixes: Vec::new(),
8232 protocol: ModuleProtocol::Subc,
8233 overlap: Default::default(),
8234 })
8235 .unwrap();
8236
8237 let deadline = Instant::now() + Duration::from_secs(5);
8238 loop {
8239 let tail = module.stderr_tail(None, None);
8240 if tail
8241 .entries
8242 .iter()
8243 .any(|entry| matches!(entry, TailEntry::ProcessStart))
8244 && tail.entries.iter().any(|entry| {
8245 matches!(
8246 entry,
8247 TailEntry::Line {
8248 truncated: true,
8249 ..
8250 }
8251 )
8252 })
8253 {
8254 break;
8255 }
8256 assert!(
8257 Instant::now() < deadline,
8258 "module did not produce a truncated line and restart boundary: {tail:?}"
8259 );
8260 sleep(Duration::from_millis(10)).await;
8261 }
8262
8263 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8264 let request = ClientControlRequest::SupervisorStderrTail {
8265 module_id: "stderr-tail-wire".to_string(),
8266 max_lines: None,
8267 max_bytes: None,
8268 };
8269 let frame = Frame::build(
8270 FrameType::Request,
8271 control_flags(),
8272 0,
8273 0,
8274 1,
8275 serde_json::to_vec(&request).unwrap(),
8276 )
8277 .unwrap();
8278 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8279 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8280 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
8281 serde_json::from_slice(&responses[0].body).unwrap()
8282 else {
8283 panic!("expected supervisor.stderr_tail response");
8284 };
8285
8286 assert!(
8287 tail.entries
8288 .iter()
8289 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
8290 "the control response lost the restart boundary"
8291 );
8292 let Some(StderrTailEntry::Line {
8293 text,
8294 truncated,
8295 at_ms,
8296 }) = tail.entries.iter().find(|entry| {
8297 matches!(
8298 entry,
8299 StderrTailEntry::Line {
8300 truncated: true,
8301 ..
8302 }
8303 )
8304 })
8305 else {
8306 panic!("the control response lost the truncated line");
8307 };
8308 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
8309 assert!(*truncated);
8310 assert!(
8311 at_ms.is_some(),
8312 "the control response lost the line's capture time"
8313 );
8314 }
8315
8316 #[tokio::test(flavor = "current_thread")]
8321 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
8322 let dir = ScratchDir::new("terminals-off-worker");
8323 let journal_path = dir.join("terminals.jsonl");
8324 let registry = Arc::new(Registry::default());
8325 let supervisor_handle = SupervisorHandle::new();
8326 let supervisor =
8327 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8328 .with_handle(supervisor_handle.clone())
8329 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
8330 let module = supervisor
8331 .spawn(ModuleSpec {
8332 module_id: "terminal-off-worker".to_string(),
8333 program: fake_aft_stub_path(),
8334 args: Vec::new(),
8335 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8336 reserved: false,
8337 reserved_prefixes: Vec::new(),
8338 protocol: ModuleProtocol::Subc,
8339 overlap: Default::default(),
8340 })
8341 .unwrap();
8342 let deadline = Instant::now() + Duration::from_secs(5);
8343 while module.terminal_history().entries.len() != 2 {
8344 assert!(Instant::now() < deadline, "module did not record two exits");
8345 sleep(Duration::from_millis(10)).await;
8346 }
8347
8348 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
8349 let handler =
8350 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
8351 let frame = Frame::build(
8352 FrameType::Request,
8353 control_flags(),
8354 0,
8355 0,
8356 1,
8357 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
8358 module_id: "terminal-off-worker".to_string(),
8359 })
8360 .unwrap(),
8361 )
8362 .unwrap();
8363 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8364 let spawned_at = std::time::Instant::now();
8365 let read = tokio::spawn({
8366 let handler = Arc::clone(&handler);
8367 async move { handler.handle_control_frame(&ctx, frame).await }
8368 });
8369 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
8372 .await
8373 .unwrap()
8374 .expect("the history read reached its pause");
8375 let elapsed = spawned_at.elapsed();
8376 assert!(
8377 elapsed < Duration::from_secs(2) && !read.is_finished(),
8378 "this task could not run while the history read was paused \
8379 (resumed after {elapsed:?}, read finished: {})",
8380 read.is_finished()
8381 );
8382
8383 drop(release);
8384 let responses = read.await.unwrap().unwrap();
8385 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8386 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
8387 panic!("expected supervisor.terminals response");
8388 };
8389 assert_eq!(terminals.entries.len(), 2);
8390 assert_eq!(terminals.journal_skipped_lines, 0);
8391 assert_eq!(terminals.journal_read_errors, 0);
8392 }
8393
8394 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
8395 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
8396 let registry = Arc::new(Registry::default());
8397 let supervisor_handle = SupervisorHandle::new();
8398 let supervisor =
8399 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
8400 .with_handle(supervisor_handle.clone());
8401 let module = supervisor
8402 .spawn(ModuleSpec {
8403 module_id: "terminal-golden".to_string(),
8404 program: fake_aft_stub_path(),
8405 args: Vec::new(),
8406 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
8407 reserved: false,
8408 reserved_prefixes: Vec::new(),
8409 protocol: ModuleProtocol::Subc,
8410 overlap: Default::default(),
8411 })
8412 .unwrap();
8413
8414 let deadline = Instant::now() + Duration::from_secs(5);
8415 while module.terminal_history().entries.len() != 2 {
8416 assert!(
8417 Instant::now() < deadline,
8418 "module did not retain two terminal exits: {:?}",
8419 module.terminal_history()
8420 );
8421 sleep(Duration::from_millis(10)).await;
8422 }
8423
8424 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
8425 let request = ClientControlRequest::SupervisorTerminals {
8426 module_id: "terminal-golden".to_string(),
8427 };
8428 let frame = Frame::build(
8429 FrameType::Request,
8430 control_flags(),
8431 0,
8432 0,
8433 1,
8434 serde_json::to_vec(&request).unwrap(),
8435 )
8436 .unwrap();
8437 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
8438 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
8439 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
8440 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
8441 panic!("expected supervisor.terminals response");
8442 };
8443 assert_eq!(terminals.entries.len(), 2);
8444 assert_eq!(terminals.dropped, 0);
8445
8446 let mut rendered = serde_json::to_value(response).unwrap();
8447 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
8450 for (index, entry) in rendered["entries"]
8451 .as_array_mut()
8452 .expect("terminal response entries array")
8453 .iter_mut()
8454 .enumerate()
8455 {
8456 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
8457 }
8458
8459 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
8460 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
8461 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
8462 if std::env::var_os("UPDATE_GOLDEN").is_some() {
8463 std::fs::write(&golden_path, &serialized).unwrap();
8464 }
8465 let expected: Value =
8466 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
8467 assert_eq!(rendered, expected);
8468 }
8469
8470 #[test]
8471 fn hello_registers_manifest_and_returns_ack() {
8472 let registry = Arc::new(Registry::default());
8473 let handler = ControlHandler::new(Arc::clone(®istry));
8474 let conn = ConnectionId::new(1);
8475
8476 let responses = handler
8477 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
8478 .unwrap();
8479
8480 assert_eq!(responses.len(), 1);
8481 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
8482 assert_eq!(responses[0].header.channel, 0);
8483 assert_eq!(responses[0].header.corr, 7);
8484 let ack = parse_ack(&responses[0]);
8485 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
8486 assert!(ack
8487 .subc_capabilities
8488 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
8489 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
8490 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
8491 assert!(ack
8492 .subc_ops
8493 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
8494 assert!(ack
8495 .subc_ops
8496 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
8497
8498 let registration = registry.get_module("aft").unwrap().unwrap();
8499 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
8500 assert_eq!(registration.state, ChannelState::Active);
8501 assert_eq!(registration.connection_id, conn);
8502 assert_eq!(registration.control_ops, module_baseline_control_ops());
8503 }
8504
8505 #[test]
8506 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
8507 let invalid_identifiers = [
8508 ("case_change", "credentials-Provider/v1"),
8509 ("leading_zero", "credentials-provider/v01"),
8510 ("trailing_hyphen", "credentials-provider-/v1"),
8511 ("consecutive_hyphens", "credentials--provider/v1"),
8512 ("uppercase", "Credentials-provider/v1"),
8513 ("missing_v", "credentials-provider/1"),
8514 ("whitespace", "credentials provider/v1"),
8515 ("zero_version", "credentials-provider/v0"),
8516 ("out_of_range_version", "credentials-provider/v4294967296"),
8517 (
8518 "overlength_name",
8519 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
8520 ),
8521 ];
8522 let mut cases = invalid_identifiers
8523 .into_iter()
8524 .map(|(name, identifier)| {
8525 (
8526 format!("identifier_{name}"),
8527 "capabilities.provides[0]".to_string(),
8528 identifier.to_string(),
8529 json!({ "provides": [identifier] }),
8530 None,
8531 )
8532 })
8533 .collect::<Vec<_>>();
8534 cases.extend([
8535 (
8536 "unknown_need".to_string(),
8537 "capabilities.requires[0].need".to_string(),
8538 "deferred".to_string(),
8539 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
8540 None,
8541 ),
8542 (
8543 "duplicate_provides".to_string(),
8544 "capabilities.provides[1]".to_string(),
8545 "credentials-provider/v1".to_string(),
8546 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
8547 None,
8548 ),
8549 (
8550 "duplicate_must_never_reach".to_string(),
8551 "capabilities.must_never_reach[1]".to_string(),
8552 "credentials-provider/v1".to_string(),
8553 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
8554 None,
8555 ),
8556 (
8557 "duplicate_requires_same_need".to_string(),
8558 "capabilities.requires[1]".to_string(),
8559 "credentials-provider/v1".to_string(),
8560 json!({ "requires": [
8561 { "capability": "credentials-provider/v1", "need": "required" },
8562 { "capability": "credentials-provider/v1", "need": "required" }
8563 ] }),
8564 None,
8565 ),
8566 (
8567 "duplicate_requires_conflicting_need".to_string(),
8568 "capabilities.requires[1]".to_string(),
8569 "credentials-provider/v1".to_string(),
8570 json!({ "requires": [
8571 { "capability": "credentials-provider/v1", "need": "required" },
8572 { "capability": "credentials-provider/v1", "need": "optional" }
8573 ] }),
8574 None,
8575 ),
8576 (
8577 "capabilities_root_pointer".to_string(),
8578 "runtime_computed[0]".to_string(),
8579 "/capabilities".to_string(),
8580 json!({}),
8581 Some(json!(["/capabilities"])),
8582 ),
8583 (
8584 "capabilities_descendant_pointer".to_string(),
8585 "runtime_computed[0]".to_string(),
8586 "/capabilities/provides".to_string(),
8587 json!({}),
8588 Some(json!(["/capabilities/provides"])),
8589 ),
8590 (
8591 "malformed_pointer_without_leading_slash".to_string(),
8592 "runtime_computed[0]".to_string(),
8593 "capabilities".to_string(),
8594 json!({}),
8595 Some(json!(["capabilities"])),
8596 ),
8597 (
8598 "malformed_pointer_escape".to_string(),
8599 "runtime_computed[0]".to_string(),
8600 "/roles/~2/tools".to_string(),
8601 json!({}),
8602 Some(json!(["/roles/~2/tools"])),
8603 ),
8604 (
8605 "unknown_capabilities_field".to_string(),
8606 "capabilities.future".to_string(),
8607 "<array>".to_string(),
8608 json!({ "future": [] }),
8609 None,
8610 ),
8611 ]);
8612
8613 for (index, (name, field, value, capabilities, runtime_computed)) in
8614 cases.into_iter().enumerate()
8615 {
8616 let registry = Arc::new(Registry::default());
8617 let handler = ControlHandler::new(Arc::clone(®istry));
8618 let response = handler
8619 .handle_control(
8620 ConnectionId::new((index + 1) as u64),
8621 capability_grammar_hello_frame(
8622 capabilities,
8623 runtime_computed,
8624 index as u64 + 1,
8625 ),
8626 )
8627 .expect("invalid HELLO returns a refusal");
8628
8629 assert_eq!(response.len(), 1, "{name} must emit one refusal");
8630 let error = parse_error(&response[0]);
8631 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
8632 let message = error["message"]
8633 .as_str()
8634 .expect("error message is a string");
8635 assert!(
8636 message.contains(&field),
8637 "{name}: field missing from {message}"
8638 );
8639 assert!(
8640 message.contains(&value),
8641 "{name}: value missing from {message}"
8642 );
8643 assert_eq!(
8644 registry
8645 .active_registration_count()
8646 .expect("registry reads"),
8647 0,
8648 "{name}: refused HELLO must not create a catalog entry"
8649 );
8650 }
8651 }
8652
8653 #[test]
8654 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
8655 let registry = Arc::new(Registry::default());
8656 let handler = ControlHandler::new(Arc::clone(®istry));
8657 let capabilities = json!({
8658 "provides": ["credentials-provider/v1"],
8659 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
8660 "must_never_reach": ["federation-transport/v1"]
8661 });
8662 let response = handler
8663 .handle_control(
8664 ConnectionId::new(99),
8665 capability_grammar_hello_frame(
8666 capabilities.clone(),
8667 Some(json!(["/roles/0/tools"])),
8668 99,
8669 ),
8670 )
8671 .expect("valid HELLO registers");
8672 assert_eq!(response[0].header.ty, FrameType::HelloAck);
8673
8674 let request = Frame::build(
8675 FrameType::Request,
8676 control_flags(),
8677 0,
8678 0,
8679 100,
8680 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8681 .expect("catalog request serializes"),
8682 )
8683 .expect("catalog request frame builds");
8684 let response = handler
8685 .handle_catalog_list(request, None)
8686 .expect("catalog list succeeds");
8687 let ClientControlResponse::CatalogList { modules, .. } =
8688 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
8689 else {
8690 panic!("catalog request must return catalog.list");
8691 };
8692 assert_eq!(modules.len(), 1);
8693 assert_eq!(
8694 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
8695 capabilities
8696 );
8697 }
8698
8699 #[test]
8700 fn catalog_list_mirrors_management_operation_description() {
8701 let registry = Arc::new(Registry::default());
8702 let handler = ControlHandler::new(Arc::clone(®istry));
8703 let description = "List managed records and return their identifiers and metadata.";
8704 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
8705 manifest.provides = vec![ProviderRole::ManagementSurface {
8706 operations: vec![ManagementOperation {
8707 name: "records.list".to_string(),
8708 kind: ManagementOperationKind::Query,
8709 description: Some(description.to_string()),
8710 }],
8711 config_schema: json!({"type": "object"}),
8712 observability: vec![ObservabilitySurface {
8713 name: "records.stats".to_string(),
8714 kind: ObservabilityKind::Snapshot,
8715 }],
8716 identity_scope: vec![IdentityScope::Project],
8717 concurrency: Concurrency::ModuleManaged,
8718 }];
8719 registry
8720 .register_with_control_ops(
8721 manifest,
8722 PROTOCOL_VERSION,
8723 ConnectionId::new(99),
8724 Vec::new(),
8725 )
8726 .expect("described management manifest registers");
8727
8728 let request = Frame::build(
8729 FrameType::Request,
8730 control_flags(),
8731 0,
8732 0,
8733 100,
8734 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
8735 .expect("catalog request serializes"),
8736 )
8737 .expect("catalog request frame builds");
8738 let response = handler
8739 .handle_catalog_list(request, None)
8740 .expect("catalog list succeeds");
8741 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
8742 assert_eq!(
8743 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
8744 "catalog.list must preserve the declared operation description verbatim"
8745 );
8746 }
8747
8748 #[test]
8749 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
8750 let registry = Arc::new(Registry::default());
8751 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8752 [("vault".to_string(), true), ("squatter".to_string(), true)],
8753 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8754 );
8755 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
8756 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8757 provides: vec!["credentials-provider/v1".to_string()],
8758 requires: Vec::new(),
8759 must_never_reach: Vec::new(),
8760 });
8761 let frame = Frame::build(
8762 FrameType::Hello,
8763 control_flags(),
8764 0,
8765 0,
8766 77,
8767 serde_json::to_vec(&ModuleHelloBody {
8768 manifest: squatter,
8769 protocol_ver: PROTOCOL_VERSION,
8770 control_ops: None,
8771 launch_nonce: None,
8772 })
8773 .expect("HELLO serializes"),
8774 )
8775 .expect("HELLO frame builds");
8776 let response = handler
8777 .handle_control(ConnectionId::new(77), frame)
8778 .expect("reserved claim receives a typed refusal");
8779 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8780 assert_eq!(
8781 registry
8782 .active_registration_count()
8783 .expect("registry reads"),
8784 0,
8785 "a reserved capability refusal must not leave a catalog entry"
8786 );
8787 }
8788
8789 #[test]
8790 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
8791 for settlement in ["timeout", "inconclusive", "drop"] {
8792 let breakers = RouteBindBreakers::default();
8793 let RouteBindAdmission::Admitted {
8794 guard: mut old,
8795 probe: false,
8796 } = breakers.admit("prov")
8797 else {
8798 panic!("ordinary relay admitted")
8799 };
8800 let RouteBindAdmission::Admitted {
8801 guard: mut opener, ..
8802 } = breakers.admit("prov")
8803 else {
8804 panic!("second relay admitted")
8805 };
8806 assert!(
8807 !opener
8808 .record_timeout(1, Duration::ZERO)
8809 .unwrap()
8810 .reopened_after_probe
8811 );
8812 let RouteBindAdmission::Admitted {
8813 guard: mut probe,
8814 probe: true,
8815 } = breakers.admit("prov")
8816 else {
8817 panic!("one cooldown probe admitted")
8818 };
8819 match settlement {
8820 "timeout" => assert!(
8821 !old.record_timeout(1, Duration::ZERO)
8822 .unwrap()
8823 .reopened_after_probe
8824 ),
8825 "inconclusive" => old.record_inconclusive(),
8826 "drop" => drop(old),
8827 _ => unreachable!(),
8828 }
8829 assert!(
8830 matches!(
8831 breakers.admit("prov"),
8832 RouteBindAdmission::Refused {
8833 probe_in_flight: true,
8834 ..
8835 }
8836 ),
8837 "{settlement} of a pre-open relay cannot release the real probe"
8838 );
8839 assert!(
8840 probe
8841 .record_timeout(1, Duration::ZERO)
8842 .unwrap()
8843 .reopened_after_probe
8844 );
8845 assert!(matches!(
8846 breakers.admit("prov"),
8847 RouteBindAdmission::Admitted { probe: true, .. }
8848 ));
8849 }
8850 let breakers = RouteBindBreakers::default();
8851 let admit = || match breakers.admit("prov") {
8852 RouteBindAdmission::Admitted { guard, .. } => guard,
8853 _ => panic!("relay admitted"),
8854 };
8855 admit().record_timeout(1, Duration::ZERO);
8856 let mut old_probe = admit();
8857 breakers.reset_for_new_module_connection("prov");
8858 admit().record_timeout(1, Duration::ZERO);
8859 let _new_probe = admit();
8860 old_probe.record_inconclusive();
8861 assert!(matches!(
8862 breakers.admit("prov"),
8863 RouteBindAdmission::Refused {
8864 probe_in_flight: true,
8865 ..
8866 }
8867 ));
8868 }
8869
8870 #[tokio::test]
8871 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
8872 for candidate in [false, true] {
8873 let registry = Arc::new(Registry::default());
8874 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8875 [("vault".to_string(), true), ("squatter".to_string(), true)],
8876 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
8877 );
8878 let conn = ConnectionId::new(77);
8879 let (ctx, mut rx) = route_ctx(conn);
8880 let initial = capability_manifest("squatter", &[], &[]);
8881 if candidate {
8882 registry
8883 .register_candidate_with_control_ops(
8884 initial.clone(),
8885 PROTOCOL_VERSION,
8886 conn,
8887 module_baseline_control_ops(),
8888 )
8889 .unwrap();
8890 handler
8891 .forwarding
8892 .register_candidate_module_connection(
8893 conn,
8894 "squatter".to_string(),
8895 PROTOCOL_VERSION,
8896 manifest_concurrency(&initial),
8897 ctx.egress.clone(),
8898 )
8899 .unwrap();
8900 } else {
8901 hello_via_sink(
8902 &handler,
8903 &ctx,
8904 &mut rx,
8905 hello_frame_with_manifest(initial.clone(), 1),
8906 )
8907 .await;
8908 }
8909 let response = handler
8910 .handle_control_frame(
8911 &ctx,
8912 catalog_update_with_capabilities_frame(
8913 2,
8914 capability_manifest("squatter", &["credentials-provider/v1"], &[])
8915 .capabilities
8916 .unwrap(),
8917 ),
8918 )
8919 .await
8920 .unwrap();
8921 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
8922 assert_eq!(
8923 registry
8924 .get_module_by_connection(conn)
8925 .unwrap()
8926 .unwrap()
8927 .manifest,
8928 initial
8929 );
8930 }
8931 }
8932
8933 #[test]
8934 fn server_describe_surfaces_required_capability_verdict_fields() {
8935 let registry = Arc::new(Registry::default());
8936 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
8937 [
8938 ("consumer".to_string(), true),
8939 ("provider".to_string(), false),
8940 ],
8941 BTreeMap::new(),
8942 );
8943 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
8944 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
8945 provides: Vec::new(),
8946 requires: vec![subc_protocol::manifest::CapabilityRequirement {
8947 capability: "credentials-provider/v1".to_string(),
8948 need: subc_protocol::manifest::CapabilityNeed::Required,
8949 }],
8950 must_never_reach: Vec::new(),
8951 });
8952 let hello = Frame::build(
8953 FrameType::Hello,
8954 control_flags(),
8955 0,
8956 0,
8957 78,
8958 serde_json::to_vec(&ModuleHelloBody {
8959 manifest: consumer,
8960 protocol_ver: PROTOCOL_VERSION,
8961 control_ops: None,
8962 launch_nonce: None,
8963 })
8964 .expect("HELLO serializes"),
8965 )
8966 .expect("HELLO frame builds");
8967 handler
8968 .handle_control(ConnectionId::new(78), hello)
8969 .expect("consumer registers");
8970 let describe = Frame::build(
8971 FrameType::Request,
8972 control_flags(),
8973 0,
8974 0,
8975 79,
8976 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
8977 .expect("request serializes"),
8978 )
8979 .expect("describe frame builds");
8980 let response = handler
8981 .handle_server_describe(describe)
8982 .expect("server.describe succeeds");
8983 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
8984 let requirement = &rendered["capability_requirements"][0];
8985 assert_eq!(requirement["consumer"], "consumer");
8986 assert_eq!(requirement["verdict"], "never_provided");
8987 assert_eq!(requirement["episode_seq"], 1);
8988 assert_eq!(requirement["config_satisfiable"], false);
8989 assert_eq!(requirement["runtime_available"], false);
8990 assert!(requirement["detail"]
8991 .as_str()
8992 .expect("detail string")
8993 .contains("credentials-provider/v1"));
8994 }
8995
8996 #[test]
8997 fn catalog_list_omits_capabilities_for_legacy_manifest() {
8998 let registry = Arc::new(Registry::default());
8999 let handler = ControlHandler::new(Arc::clone(®istry));
9000 let hello = handler
9001 .handle_control(
9002 ConnectionId::new(101),
9003 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
9004 )
9005 .expect("legacy HELLO registers");
9006 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9007
9008 let request = Frame::build(
9009 FrameType::Request,
9010 control_flags(),
9011 0,
9012 0,
9013 102,
9014 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9015 .expect("catalog request serializes"),
9016 )
9017 .expect("catalog request frame builds");
9018 let response = handler
9019 .handle_catalog_list(request, None)
9020 .expect("catalog list succeeds");
9021 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9022 assert!(
9023 body["modules"][0].get("capabilities").is_none(),
9024 "legacy manifest must retain an absent capabilities field on catalog.list"
9025 );
9026 }
9027
9028 #[test]
9029 fn hello_ack_omits_storage_when_no_storage_config() {
9030 let registry = Arc::new(Registry::default());
9031 let handler = ControlHandler::new(Arc::clone(®istry));
9032 let responses = handler
9033 .handle_control(
9034 ConnectionId::new(1),
9035 hello_frame("aft", PROTOCOL_VERSION, 7),
9036 )
9037 .unwrap();
9038 let ack = parse_ack(&responses[0]);
9039 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
9040 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
9041 }
9042
9043 #[tokio::test]
9044 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
9045 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
9046 let registry = Arc::new(Registry::default());
9047 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
9048 let responses = handler
9049 .handle_control(
9050 ConnectionId::new(1),
9051 hello_frame("aft", PROTOCOL_VERSION, 7),
9052 )
9053 .unwrap();
9054 let ack = parse_ack(&responses[0]);
9055 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
9056
9057 let described = handler
9058 .handle_control_frame(
9059 &route_ctx(ConnectionId::new(2)).0,
9060 Frame::build(
9061 FrameType::Request,
9062 control_flags(),
9063 0,
9064 0,
9065 9,
9066 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
9067 )
9068 .unwrap(),
9069 )
9070 .await
9071 .unwrap();
9072 let ClientControlResponse::ServerDescribe { machine_id, .. } =
9073 serde_json::from_slice(&described[0].body).unwrap()
9074 else {
9075 panic!("server.describe answered with another shape");
9076 };
9077 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
9078 }
9079
9080 #[test]
9081 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
9082 let registry = Arc::new(Registry::default());
9085 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
9086 crate::daemon_config::StorageConfig::Sqlite {
9087 data_home: std::path::PathBuf::from("/data"),
9088 },
9089 ));
9090
9091 let responses = handler
9092 .handle_control(
9093 ConnectionId::new(1),
9094 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
9095 )
9096 .unwrap();
9097 let ack = parse_ack(&responses[0]);
9098 assert_eq!(
9099 ack.storage,
9100 Some(serde_json::json!({
9101 "module_id": "alfonso-routing",
9102 "storage_namespace": "default",
9103 "isolation": { "kind": "module" },
9104 "backend": {
9105 "backend": "sqlite",
9106 "path": "/data/cortexkit/alfonso-routing/store.db"
9107 }
9108 })),
9109 "the delivered descriptor is the module's own sqlite store path"
9110 );
9111 }
9112
9113 #[test]
9114 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
9115 let registry = Arc::new(Registry::default());
9116 let handler = ControlHandler::new(Arc::clone(®istry));
9117 let conn = ConnectionId::new(1);
9118 let responses = handler
9119 .handle_control(
9120 conn,
9121 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9122 )
9123 .unwrap();
9124 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9125 let registration = registry.get_module("aft").unwrap().unwrap();
9126 assert_eq!(registration.control_ops, module_baseline_control_ops());
9127
9128 let frame =
9129 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
9130 assert!(handler
9131 .guard_module_control_op(&frame, "aft", "route.bind")
9132 .unwrap()
9133 .is_none());
9134 let error = handler
9135 .guard_module_control_op(&frame, "aft", "test.synthetic")
9136 .unwrap()
9137 .expect("synthetic ungranted op should be rejected");
9138 assert_eq!(error.header.ty, FrameType::Error);
9139 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
9140 }
9141
9142 #[test]
9143 fn hello_control_ops_some_adds_optional_grants() {
9144 let registry = Arc::new(Registry::default());
9145 let handler = ControlHandler::new(Arc::clone(®istry));
9146 handler
9147 .handle_control(
9148 ConnectionId::new(1),
9149 hello_frame_with_control_ops(
9150 "aft",
9151 PROTOCOL_VERSION,
9152 7,
9153 Some(vec![
9154 "future.synthetic".to_string(),
9155 "route.bind".to_string(),
9156 ]),
9157 ),
9158 )
9159 .unwrap();
9160 let registration = registry.get_module("aft").unwrap().unwrap();
9161 assert_eq!(
9162 registration.control_ops,
9163 vec![
9164 "route.bind".to_string(),
9165 "route.status".to_string(),
9166 "future.synthetic".to_string(),
9167 ]
9168 );
9169 let frame =
9170 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
9171 assert!(handler
9172 .guard_module_control_op(&frame, "aft", "future.synthetic")
9173 .unwrap()
9174 .is_none());
9175 }
9176
9177 #[tokio::test]
9178 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
9179 let registry = Arc::new(Registry::default());
9180 let forwarding = Arc::new(ForwardingTable::default());
9181 let handler =
9182 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9183 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
9184 hello_via_sink(
9185 &handler,
9186 &module_ctx,
9187 &mut module_rx,
9188 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
9189 )
9190 .await;
9191
9192 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
9193 let responses = handler
9194 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
9195 .await
9196 .unwrap();
9197 assert_eq!(responses.len(), 1);
9198 assert_eq!(responses[0].header.ty, FrameType::Error);
9199 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
9200 assert!(module_rx.try_recv().is_err());
9201 }
9202
9203 #[tokio::test]
9204 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
9205 let registry = Arc::new(Registry::default());
9206 let forwarding = Arc::new(ForwardingTable::default());
9207 let handler =
9208 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9209 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
9210 hello_via_sink(
9211 &handler,
9212 &module_ctx,
9213 &mut module_rx,
9214 hello_frame_with_control_ops(
9215 "aft",
9216 PROTOCOL_VERSION,
9217 7,
9218 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
9219 ),
9220 )
9221 .await;
9222
9223 let project_root = unique_project_root("demux");
9224 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
9225 let route_handler = handler.clone();
9226 let route_task = tokio::spawn(async move {
9227 route_handler
9228 .handle_control_frame(
9229 &route_client_ctx,
9230 route_open_frame(100, "aft", project_root),
9231 )
9232 .await
9233 .unwrap()
9234 });
9235 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9236 .await
9237 .unwrap()
9238 .unwrap();
9239 assert!(matches!(
9240 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
9241 ModuleControlRequest::RouteBind { .. }
9242 ));
9243
9244 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
9245 let health_handler = handler.clone();
9246 let health_task = tokio::spawn(async move {
9247 health_handler
9248 .handle_control_frame(
9249 &health_client_ctx,
9250 supervisor_health_probe_frame(101, "aft"),
9251 )
9252 .await
9253 .unwrap()
9254 });
9255 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9256 .await
9257 .unwrap()
9258 .unwrap();
9259 assert_eq!(
9260 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
9261 ModuleControlRequest::HealthCheck {}
9262 );
9263
9264 handler
9265 .handle_control_frame(
9266 &module_ctx,
9267 health_response(health_frame.header.corr, HealthStatus::Degraded),
9268 )
9269 .await
9270 .unwrap();
9271 let health_response = health_task.await.unwrap();
9272 assert_eq!(health_response.len(), 1);
9273 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
9274 ClientControlResponse::SupervisorHealthProbe {
9275 module_id,
9276 status,
9277 detail,
9278 metrics,
9279 } => {
9280 assert_eq!(module_id, "aft");
9281 assert_eq!(status, HealthStatus::Degraded);
9282 assert_eq!(detail.as_deref(), Some("warming"));
9283 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
9284 }
9285 other => panic!("unexpected health response: {other:?}"),
9286 }
9287
9288 handler
9289 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
9290 .await
9291 .unwrap();
9292 let route_response = route_task.await.unwrap();
9293 assert!(route_response.is_empty());
9294 let published = route_client_rx.recv().await.unwrap();
9295 assert!(matches!(
9296 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
9297 ClientControlResponse::RouteOpen { .. }
9298 ));
9299 }
9300
9301 async fn relay_route_open(
9306 handler: &ControlHandler,
9307 client_connection: ConnectionId,
9308 client_egress: &FrameSink,
9309 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9310 corr: u64,
9311 module_id: &str,
9312 project_root_label: &str,
9313 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
9314 let ctx = RouteCtx {
9315 connection_id: client_connection,
9316 egress: client_egress.clone(),
9317 };
9318 let handler = handler.clone();
9319 let project_root = unique_project_root(project_root_label);
9320 let module_id = module_id.to_string();
9321 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
9322 let task = tokio::spawn(async move {
9323 let _guard = tracing::dispatcher::set_default(&dispatch);
9324 handler
9325 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
9326 .await
9327 .unwrap()
9328 });
9329 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
9330 .await
9331 .expect("module receives the relayed route.bind")
9332 .expect("module egress is open");
9333 (task, bind.frame)
9334 }
9335
9336 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
9337 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
9338 ModuleControlRequest::RouteBind {
9339 route_channel,
9340 epoch,
9341 ..
9342 } => (route_channel, epoch),
9343 other => panic!("expected a route.bind request, got {other:?}"),
9344 }
9345 }
9346
9347 fn published_route(frame: &Frame) -> (u16, u32) {
9348 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
9349 ClientControlResponse::RouteOpen {
9350 route_channel,
9351 route_epoch,
9352 } => (route_channel, route_epoch),
9353 other => panic!("expected a route.open response, got {other:?}"),
9354 }
9355 }
9356
9357 #[tokio::test]
9377 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
9378 let registry = Arc::new(Registry::default());
9379 let forwarding = Arc::new(ForwardingTable::default());
9380 let handler =
9381 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
9382
9383 let module_connection = ConnectionId::new(30);
9384 let (module_ctx, mut module_rx) = route_ctx(module_connection);
9385 hello_via_sink(
9386 &handler,
9387 &module_ctx,
9388 &mut module_rx,
9389 hello_frame("aft", PROTOCOL_VERSION, 7),
9390 )
9391 .await;
9392
9393 let dying_client = ConnectionId::new(31);
9394 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
9395
9396 let (first_task, first_bind) = relay_route_open(
9399 &handler,
9400 dying_client,
9401 &dying_ctx.egress,
9402 &mut module_rx,
9403 100,
9404 "aft",
9405 "closing-first",
9406 )
9407 .await;
9408 handler
9409 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
9410 .await
9411 .unwrap();
9412 assert!(first_task.await.unwrap().is_empty());
9413 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
9414
9415 let (second_task, second_bind) = relay_route_open(
9417 &handler,
9418 dying_client,
9419 &dying_ctx.egress,
9420 &mut module_rx,
9421 101,
9422 "aft",
9423 "closing-second",
9424 )
9425 .await;
9426 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
9427
9428 assert!(forwarding
9431 .escalate_client_delivery_failure(
9432 dying_client,
9433 first_channel,
9434 first_epoch,
9435 CloseReason::new(
9436 "module_to_client_delivery_failed",
9437 "client egress refused a module frame",
9438 ),
9439 crate::forwarding::UndeliveredFrame {
9440 module_id: None,
9441 sink: &dying_ctx.egress,
9442 },
9443 )
9444 .unwrap());
9445 assert!(!dying_ctx.egress.is_closed());
9446
9447 let ack = handler
9449 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
9450 .await;
9451 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
9452 if module_loop_error.is_some() {
9453 handler.cleanup_connection(module_connection).unwrap();
9457 }
9458 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
9462 .await
9463 .ok()
9464 .flatten();
9465
9466 assert!(
9468 registry
9469 .get_module_by_connection(module_connection)
9470 .unwrap()
9471 .is_some(),
9472 "one client's closing connection ended the shared module connection: \
9473 {module_loop_error:?}"
9474 );
9475 let cotenant = ConnectionId::new(32);
9477 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
9478 let (cotenant_task, cotenant_bind) = relay_route_open(
9479 &handler,
9480 cotenant,
9481 &cotenant_ctx.egress,
9482 &mut module_rx,
9483 102,
9484 "aft",
9485 "closing-cotenant",
9486 )
9487 .await;
9488 handler
9489 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
9490 .await
9491 .unwrap();
9492 assert!(cotenant_task.await.unwrap().is_empty());
9493 let (cotenant_channel, cotenant_epoch) =
9494 published_route(&cotenant_rx.recv().await.unwrap());
9495 assert!(matches!(
9496 forwarding
9497 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
9498 .unwrap(),
9499 DataRoute::Client(DataRouteState::Bound(_))
9500 ));
9501
9502 let goodbye = post_ack_module_frame
9505 .expect("module receives a GOODBYE for the abandoned route channel");
9506 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9507 assert_eq!(goodbye.header.channel, abandoned_channel);
9508 assert_eq!(goodbye.header.epoch, abandoned_epoch);
9509
9510 assert!(dying_rx.try_recv().is_err());
9514 let second_response = second_task.await.unwrap();
9515 assert_eq!(second_response.len(), 1);
9516 assert_eq!(
9517 parse_error(&second_response[0])["code"],
9518 "target_unavailable"
9519 );
9520 }
9521
9522 async fn catalog_snapshot(
9525 handler: &ControlHandler,
9526 client_ctx: &RouteCtx,
9527 corr: u64,
9528 ) -> (u64, Vec<(String, bool)>) {
9529 let body =
9530 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
9531 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
9532 let reply = handler
9533 .handle_control_frame(client_ctx, request)
9534 .await
9535 .unwrap()
9536 .pop()
9537 .unwrap();
9538 let ClientControlResponse::CatalogList {
9539 generation,
9540 modules,
9541 ..
9542 } = serde_json::from_slice(&reply.body).unwrap()
9543 else {
9544 panic!("not a catalog.list reply");
9545 };
9546 (
9547 generation,
9548 modules
9549 .into_iter()
9550 .map(|entry| (entry.module_id, entry.ready))
9551 .collect(),
9552 )
9553 }
9554
9555 fn hello_held_before_forwarding() -> (
9561 ControlHandler,
9562 Arc<std::sync::Barrier>,
9563 std::thread::JoinHandle<Vec<Frame>>,
9564 RouteCtx,
9565 mpsc::Receiver<crate::router::OutboundFrame>,
9566 ) {
9567 let hold = Arc::new(std::sync::Barrier::new(2));
9568 let handler = ControlHandler::with_forwarding(
9569 Arc::new(Registry::default()),
9570 Arc::new(ForwardingTable::default()),
9571 )
9572 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
9573 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9574 let hello = {
9575 let handler = handler.clone();
9576 let module_ctx = module_ctx.clone();
9577 std::thread::spawn(move || {
9578 handler
9579 .handle_hello(
9580 module_ctx.connection_id,
9581 Some(module_ctx.egress.clone()),
9582 hello_frame("aft", PROTOCOL_VERSION, 7),
9583 )
9584 .unwrap()
9585 })
9586 };
9587 (handler, hold, hello, module_ctx, module_rx)
9588 }
9589
9590 #[tokio::test]
9599 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
9600 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
9601 let client = ConnectionId::new(41);
9602 let (client_ctx, mut client_rx) = route_ctx(client);
9603
9604 hold.wait();
9605 assert!(handler.registry.get_module("aft").unwrap().is_some());
9607 assert!(!handler
9608 .forwarding
9609 .has_live_module_connection("aft")
9610 .unwrap());
9611 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
9612 assert!(
9613 listed.is_empty(),
9614 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
9615 );
9616 let refused = handler
9617 .handle_control_frame(
9618 &client_ctx,
9619 route_open_frame(301, "aft", unique_project_root("listed-window")),
9620 )
9621 .await
9622 .unwrap();
9623 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
9624 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
9625 let duplicate = handler
9626 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
9627 .await
9628 .unwrap();
9629 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
9630
9631 hold.wait();
9632 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9633 let ack = module_rx.try_recv().unwrap().frame;
9634 assert_eq!(ack.header.ty, FrameType::HelloAck);
9635 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9636 assert_eq!(listed, vec![("aft".to_string(), true)]);
9637
9638 let (open_task, bind) = relay_route_open(
9639 &handler,
9640 client,
9641 &client_ctx.egress,
9642 &mut module_rx,
9643 303,
9644 "aft",
9645 "listed-after",
9646 )
9647 .await;
9648 assert!(handler
9649 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9650 .await
9651 .unwrap()
9652 .is_empty());
9653 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
9654 published_route(&client_rx.try_recv().unwrap().frame);
9655 }
9656
9657 #[tokio::test]
9664 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
9665 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
9666 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9667 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9668
9669 hold.wait();
9670 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
9671 assert!(listed.is_empty());
9672 assert_eq!(
9673 during, before,
9674 "the generation moved while the module was not listed"
9675 );
9676
9677 hold.wait();
9678 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
9679 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
9680 assert_eq!(listed, vec![("aft".to_string(), true)]);
9681 assert_eq!(
9682 after,
9683 before + 1,
9684 "the module appeared without exactly one generation bump"
9685 );
9686 }
9687
9688 #[tokio::test]
9693 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
9694 let registry = Arc::new(Registry::default());
9695 let handler = ControlHandler::with_forwarding(
9696 Arc::clone(®istry),
9697 Arc::new(ForwardingTable::default()),
9698 );
9699 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
9700 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
9701 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
9702 drop(module_rx);
9703
9704 let replies = handler
9705 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
9706 .await
9707 .unwrap();
9708 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
9709 assert!(registry.get_module("aft").unwrap().is_none());
9710 assert!(registry
9711 .get_module_by_connection(module_ctx.connection_id)
9712 .unwrap()
9713 .is_none());
9714 assert_eq!(
9715 catalog_snapshot(&handler, &client_ctx, 301).await,
9716 (before, Vec::new())
9717 );
9718 }
9719
9720 async fn route_open_held_after_bind_send() -> (
9724 ControlHandler,
9725 Arc<tokio::sync::Barrier>,
9726 RouteCtx,
9727 mpsc::Receiver<crate::router::OutboundFrame>,
9728 ) {
9729 let hold = Arc::new(tokio::sync::Barrier::new(2));
9730 let handler = ControlHandler::with_forwarding(
9731 Arc::new(Registry::default()),
9732 Arc::new(ForwardingTable::default()),
9733 )
9734 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
9735 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
9736 hello_via_sink(
9737 &handler,
9738 &module_ctx,
9739 &mut module_rx,
9740 hello_frame("aft", PROTOCOL_VERSION, 7),
9741 )
9742 .await;
9743 (handler, hold, module_ctx, module_rx)
9744 }
9745
9746 #[tokio::test]
9753 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
9754 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9755 let client = ConnectionId::new(41);
9756 let (client_ctx, mut client_rx) = route_ctx(client);
9757
9758 let (open_task, bind) = relay_route_open(
9759 &handler,
9760 client,
9761 &client_ctx.egress,
9762 &mut module_rx,
9763 200,
9764 "aft",
9765 "acked-before-mark",
9766 )
9767 .await;
9768 let (module_channel, module_epoch) = route_bind_channel(&bind);
9769 hold.wait().await;
9771
9772 assert!(handler
9774 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9775 .await
9776 .unwrap()
9777 .is_empty());
9778 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
9779
9780 hold.wait().await;
9781 assert!(
9782 open_task.await.unwrap().is_empty(),
9783 "route.open answers accepted"
9784 );
9785
9786 match module_rx.try_recv() {
9789 Err(mpsc::error::TryRecvError::Empty) => {}
9790 Ok(outbound) => panic!(
9791 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
9792 outbound.frame.header.ty,
9793 outbound.frame.header.channel,
9794 outbound.frame.header.epoch
9795 ),
9796 Err(err) => panic!("module egress closed: {err}"),
9797 }
9798
9799 let router = Router::with_control_handler(Arc::new(handler.clone()));
9802 let request = Frame::build(
9803 FrameType::Request,
9804 Flags::new(true, Priority::Interactive, false),
9805 client_channel,
9806 client_epoch,
9807 201,
9808 b"first request".to_vec(),
9809 )
9810 .unwrap();
9811 router
9812 .route_for_connection(&client_ctx, request)
9813 .await
9814 .unwrap();
9815 let delivered = module_rx.try_recv().unwrap().frame;
9816 assert_eq!(delivered.header.ty, FrameType::Request);
9817 assert_eq!(
9818 (delivered.header.channel, delivered.header.epoch),
9819 (module_channel, module_epoch)
9820 );
9821 assert_eq!(delivered.body, b"first request");
9822 }
9823
9824 #[tokio::test]
9830 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
9831 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9832 let client = ConnectionId::new(42);
9833 let (client_ctx, _client_rx) = route_ctx(client);
9834
9835 let (open_task, bind) = relay_route_open(
9836 &handler,
9837 client,
9838 &client_ctx.egress,
9839 &mut module_rx,
9840 300,
9841 "aft",
9842 "released-before-mark",
9843 )
9844 .await;
9845 let (module_channel, module_epoch) = route_bind_channel(&bind);
9846 hold.wait().await;
9847
9848 handler.cleanup_connection(client).unwrap();
9849 assert!(
9850 module_rx.try_recv().is_err(),
9851 "client teardown must not GOODBYE a bind route.open is still queueing"
9852 );
9853
9854 hold.wait().await;
9855 let response = open_task.await.unwrap();
9856 assert_eq!(response.len(), 1);
9857 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9858
9859 let goodbye = module_rx
9860 .try_recv()
9861 .expect("module is told to drop the binding the daemon abandoned")
9862 .frame;
9863 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9864 assert_eq!(
9865 (goodbye.header.channel, goodbye.header.epoch),
9866 (module_channel, module_epoch)
9867 );
9868 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9869 }
9870
9871 #[tokio::test]
9876 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
9877 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
9878 let client = ConnectionId::new(43);
9879 let (client_ctx, client_rx) = route_ctx(client);
9880
9881 let (open_task, bind) = relay_route_open(
9882 &handler,
9883 client,
9884 &client_ctx.egress,
9885 &mut module_rx,
9886 400,
9887 "aft",
9888 "gone-client-before-mark",
9889 )
9890 .await;
9891 let (module_channel, module_epoch) = route_bind_channel(&bind);
9892 hold.wait().await;
9893
9894 drop(client_rx);
9895 assert!(client_ctx.egress.is_closed());
9896 handler
9897 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
9898 .await
9899 .unwrap();
9900
9901 hold.wait().await;
9902 let response = open_task.await.unwrap();
9903 assert_eq!(response.len(), 1);
9904 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
9905
9906 let goodbye = module_rx
9907 .try_recv()
9908 .expect("module is told to drop the binding it acked")
9909 .frame;
9910 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
9911 assert_eq!(
9912 (goodbye.header.channel, goodbye.header.epoch),
9913 (module_channel, module_epoch)
9914 );
9915 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
9916 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
9917 }
9918
9919 #[test]
9926 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
9927 let handler = ControlHandler::default();
9928 let module_connection = ConnectionId::new(30);
9929 let client_connection = ConnectionId::new(31);
9930
9931 handler
9932 .refuse_to_end_module_connection_for_a_client(
9933 module_connection,
9934 77,
9935 ForwardingError::ConnectionClosing {
9936 connection_id: client_connection,
9937 },
9938 )
9939 .expect("a closing client must never end the module connection");
9940
9941 assert!(matches!(
9942 handler.refuse_to_end_module_connection_for_a_client(
9943 module_connection,
9944 78,
9945 ForwardingError::ConnectionClosing {
9946 connection_id: module_connection,
9947 },
9948 ),
9949 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
9950 connection_id
9951 })) if connection_id == module_connection
9952 ));
9953 assert!(matches!(
9954 handler.refuse_to_end_module_connection_for_a_client(
9955 module_connection,
9956 79,
9957 ForwardingError::Poisoned,
9958 ),
9959 Err(RouterError::Forwarding(ForwardingError::Poisoned))
9960 ));
9961 assert!(matches!(
9962 handler.refuse_to_end_module_connection_for_a_client(
9963 module_connection,
9964 80,
9965 ForwardingError::StaleModuleEndpoint,
9966 ),
9967 Err(RouterError::Forwarding(
9968 ForwardingError::StaleModuleEndpoint
9969 ))
9970 ));
9971 }
9972
9973 #[tokio::test]
9986 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
9987 let registry = Arc::new(Registry::default());
9988 let forwarding = Arc::new(ForwardingTable::default());
9989 let supervisor = SupervisorHandle::new();
9990 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
9991 let handler =
9992 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
9993 .with_supervisor(supervisor);
9994
9995 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
9996 hello_via_sink(
9997 &handler,
9998 &target_ctx,
9999 &mut target_rx,
10000 hello_frame("target", PROTOCOL_VERSION, 1),
10001 )
10002 .await;
10003
10004 let wrong_nonce = handler
10008 .handle_control_frame(
10009 &route_ctx(ConnectionId::new(91)).0,
10010 route_open_frame_with_admission_facts(
10011 20,
10012 "target",
10013 unique_project_root("admission-facts"),
10014 Some(subc_control::ConsumerIdentity {
10015 module_id: "fed".to_string(),
10016 launch_nonce: "not-the-real-nonce".to_string(),
10017 }),
10018 None,
10019 ),
10020 )
10021 .await
10022 .unwrap();
10023 assert_eq!(
10024 parse_error(&wrong_nonce[0])["code"],
10025 "bad_consumer_identity",
10026 "a mismatched launch nonce must be refused, not stamped Reserved"
10027 );
10028
10029 let never_spawned = handler
10033 .handle_control_frame(
10034 &route_ctx(ConnectionId::new(92)).0,
10035 route_open_frame_with_admission_facts(
10036 21,
10037 "target",
10038 unique_project_root("admission-facts"),
10039 Some(subc_control::ConsumerIdentity {
10040 module_id: "never-spawned".to_string(),
10041 launch_nonce: "any-nonce".to_string(),
10042 }),
10043 None,
10044 ),
10045 )
10046 .await
10047 .unwrap();
10048 assert_eq!(
10049 parse_error(&never_spawned[0])["code"],
10050 "bad_consumer_identity",
10051 "an unspawned module_id must be refused rather than accepted for lack of a record"
10052 );
10053 }
10054
10055 #[tokio::test]
10078 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
10079 let registry = Arc::new(Registry::default());
10080 let forwarding = Arc::new(ForwardingTable::default());
10081 let supervisor = SupervisorHandle::new();
10082 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10083 let handler =
10084 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10085 .with_supervisor(supervisor);
10086
10087 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
10088 hello_via_sink(
10089 &handler,
10090 &target_ctx,
10091 &mut target_rx,
10092 hello_frame("target", PROTOCOL_VERSION, 1),
10093 )
10094 .await;
10095
10096 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
10097 let route_handler = handler.clone();
10098 let route_task = tokio::spawn(async move {
10099 route_handler
10100 .handle_control_frame(
10101 &client_ctx,
10102 route_open_frame_with_admission_facts(
10103 30,
10104 "target",
10105 unique_project_root("admission-facts"),
10106 Some(subc_control::ConsumerIdentity {
10107 module_id: "fed".to_string(),
10108 launch_nonce: "fed-nonce".to_string(),
10109 }),
10110 None,
10111 ),
10112 )
10113 .await
10114 .unwrap()
10115 });
10116
10117 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10124 .await
10125 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
10126 .expect("module control channel closed before route.bind");
10127 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10128 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
10129 panic!("expected route.bind")
10130 };
10131 assert_eq!(
10132 principal,
10133 Some(Principal::Reserved {
10134 module_id: "fed".to_string()
10135 }),
10136 "a correctly attested consumer must be stamped Reserved for its own id"
10137 );
10138
10139 handler
10140 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10141 .await
10142 .unwrap();
10143 assert!(route_task.await.unwrap().is_empty());
10144 assert!(
10145 matches!(
10146 serde_json::from_slice::<ClientControlResponse>(
10147 &client_rx.recv().await.unwrap().body
10148 )
10149 .unwrap(),
10150 ClientControlResponse::RouteOpen { .. }
10151 ),
10152 "the route must actually open, not merely avoid an error"
10153 );
10154 }
10155
10156 #[tokio::test(start_paused = true)]
10157 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
10158 let registry = Arc::new(Registry::default());
10159 let forwarding = Arc::new(ForwardingTable::default());
10160 let supervisor = SupervisorHandle::new();
10161 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10162 let handler =
10163 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10164 .with_supervisor(supervisor);
10165
10166 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
10167 hello_via_sink(
10168 &handler,
10169 &target_ctx,
10170 &mut target_rx,
10171 hello_frame("target", PROTOCOL_VERSION, 1),
10172 )
10173 .await;
10174
10175 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
10176 let direct_handler = handler.clone();
10177 let direct_open = tokio::spawn(async move {
10178 direct_handler
10179 .handle_control_frame(
10180 &direct_ctx,
10181 route_open_frame(2, "target", unique_project_root("route-census-direct")),
10182 )
10183 .await
10184 .unwrap()
10185 });
10186 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10187 .await
10188 .expect("no direct route.bind within 5s")
10189 .expect("target control channel closed before direct route.bind");
10190 handler
10191 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
10192 .await
10193 .unwrap();
10194 assert!(direct_open.await.unwrap().is_empty());
10195 let _ = direct_rx.recv().await.unwrap();
10196
10197 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
10198 let reserved_handler = handler.clone();
10199 let reserved_open = tokio::spawn(async move {
10200 reserved_handler
10201 .handle_control_frame(
10202 &reserved_ctx,
10203 route_open_frame_with_admission_facts(
10204 3,
10205 "target",
10206 unique_project_root("admission-facts"),
10207 Some(ConsumerIdentity {
10208 module_id: "fed".to_string(),
10209 launch_nonce: "fed-nonce".to_string(),
10210 }),
10211 None,
10212 ),
10213 )
10214 .await
10215 .unwrap()
10216 });
10217 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10218 .await
10219 .expect("no reserved route.bind within 5s")
10220 .expect("target control channel closed before reserved route.bind");
10221 handler
10222 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
10223 .await
10224 .unwrap();
10225 assert!(reserved_open.await.unwrap().is_empty());
10226 let _ = reserved_rx.recv().await.unwrap();
10227
10228 forwarding
10229 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
10230 .unwrap();
10231 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
10232 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
10233 module_id: Some("target".to_string()),
10234 })
10235 .unwrap();
10236 let census_frame =
10237 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
10238 let response = handler
10239 .handle_control_frame(&census_ctx, census_frame)
10240 .await
10241 .unwrap()
10242 .pop()
10243 .unwrap();
10244 let actual: Value = serde_json::from_slice(&response.body).unwrap();
10245 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
10246 assert!(matches!(
10247 decoded,
10248 ClientControlResponse::SupervisorRoutes { .. }
10249 ));
10250 let routes = actual["modules"][0]["routes"].as_array().unwrap();
10251 assert_eq!(routes.len(), 2);
10252 assert!(routes.iter().all(|route| route["draining"] == true));
10253 assert!(
10256 routes.iter().all(|route| route["drain_reason"] == "reload"),
10257 "draining routes must name the drain's reason: {routes:?}"
10258 );
10259 assert!(routes.iter().any(|route| {
10260 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
10261 }));
10262 assert!(routes.iter().any(|route| {
10263 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
10264 }));
10265
10266 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10267 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
10268 if std::env::var_os("UPDATE_GOLDEN").is_some() {
10269 std::fs::write(
10270 &golden_path,
10271 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
10272 )
10273 .unwrap();
10274 }
10275 let expected: Value =
10276 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
10277 assert_eq!(actual, expected);
10278 }
10279
10280 async fn query_live_roots(
10281 handler: &ControlHandler,
10282 module_ctx: &RouteCtx,
10283 ) -> ModuleControlResponseToModule {
10284 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
10285 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
10286 let response = handler
10287 .handle_control_frame(module_ctx, frame)
10288 .await
10289 .unwrap()
10290 .pop()
10291 .unwrap();
10292 serde_json::from_slice(&response.body).unwrap()
10293 }
10294
10295 #[tokio::test(start_paused = true)]
10296 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
10297 let registry = Arc::new(Registry::default());
10298 let forwarding = Arc::new(ForwardingTable::default());
10299 let handler = ControlHandler::with_forwarding(registry, forwarding);
10300 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
10301 hello_via_sink(
10302 &handler,
10303 &target_ctx,
10304 &mut target_rx,
10305 hello_frame("target", PROTOCOL_VERSION, 1),
10306 )
10307 .await;
10308 let root = unique_project_root("live-roots-known");
10309 let path = ProjectRootId::from_path_allowing_missing(root.path())
10310 .unwrap()
10311 .as_path()
10312 .to_path_buf();
10313 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
10314 let open_handler = handler.clone();
10315 let opened = tokio::spawn(async move {
10316 open_handler
10317 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
10318 .await
10319 .unwrap()
10320 });
10321 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10322 .await
10323 .unwrap()
10324 .unwrap();
10325 handler
10326 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
10327 .await
10328 .unwrap();
10329 assert!(opened.await.unwrap().is_empty());
10330 let _ = client_rx.recv().await.unwrap();
10331
10332 let root = unique_project_root("live-roots-pending");
10333 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
10334 .unwrap()
10335 .as_path()
10336 .to_path_buf();
10337 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
10338 let open_handler = handler.clone();
10339 let pending = tokio::spawn(async move {
10340 open_handler
10341 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
10342 .await
10343 .unwrap()
10344 });
10345 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
10346 .await
10347 .unwrap()
10348 .unwrap();
10349 let actual = query_live_roots(&handler, &target_ctx).await;
10350 let ModuleControlResponseToModule::LiveRoots {
10351 roots,
10352 unknown_root_bindings,
10353 total_bindings,
10354 } = actual
10355 else {
10356 panic!("expected live roots")
10357 };
10358 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
10359 assert_eq!(unknown_root_bindings, 0);
10360 assert_eq!(
10361 roots.len(),
10362 2,
10363 "root-known arm must retain each canonical root"
10364 );
10365 assert_eq!(
10366 total_bindings,
10367 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10368 );
10369 let counts = roots
10370 .iter()
10371 .map(|root| (root.project_root.clone(), root.bound, root.pending))
10372 .collect::<Vec<_>>();
10373 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
10374 expected.sort_by(|a, b| a.0.cmp(&b.0));
10375 assert_eq!(
10376 counts, expected,
10377 "roots must sort by path and count pending separately"
10378 );
10379 handler
10380 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
10381 .await
10382 .unwrap();
10383 assert!(pending.await.unwrap().is_empty());
10384 }
10385
10386 #[tokio::test(start_paused = true)]
10387 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
10388 let forwarding = Arc::new(ForwardingTable::default());
10389 let handler =
10390 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10391 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
10392 hello_via_sink(
10393 &handler,
10394 &target_ctx,
10395 &mut target_rx,
10396 hello_frame("target", PROTOCOL_VERSION, 1),
10397 )
10398 .await;
10399 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
10400 let pending = forwarding
10401 .begin_route_bind_relay_for_test(
10402 client_ctx.connection_id,
10403 client_ctx.egress.clone(),
10404 2,
10405 "target",
10406 )
10407 .unwrap();
10408 forwarding
10409 .complete_pending_relay(
10410 target_ctx.connection_id,
10411 pending.corr,
10412 RouteBindRelayOutcome::Accepted,
10413 )
10414 .unwrap();
10415 let actual = query_live_roots(&handler, &target_ctx).await;
10416 let ModuleControlResponseToModule::LiveRoots {
10417 roots,
10418 unknown_root_bindings,
10419 total_bindings,
10420 } = actual
10421 else {
10422 panic!("expected live roots")
10423 };
10424 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
10425 assert_eq!(
10426 unknown_root_bindings, 1,
10427 "unknown-root arm must not read as no bindings"
10428 );
10429 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
10430 assert_eq!(
10431 total_bindings,
10432 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10433 );
10434 }
10435
10436 #[tokio::test(start_paused = true)]
10444 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
10445 let forwarding = Arc::new(ForwardingTable::default());
10446 let handler =
10447 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
10448 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
10449 let replies = handler
10450 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
10451 .await
10452 .unwrap();
10453 let queued_by_hello = module_rx.len();
10454
10455 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
10456 let open_handler = handler.clone();
10457 let open = tokio::spawn(async move {
10458 open_handler
10459 .handle_control_frame(
10460 &client_ctx,
10461 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
10462 )
10463 .await
10464 .unwrap()
10465 });
10466 let mut spins = 0;
10468 while module_rx.len() == queued_by_hello {
10469 spins += 1;
10470 assert!(spins < 10_000, "route.open never queued a route.bind");
10471 tokio::task::yield_now().await;
10472 }
10473
10474 for reply in replies {
10476 module_ctx.egress.send(reply).await.unwrap();
10477 }
10478
10479 let first = module_rx.recv().await.unwrap().frame;
10480 assert_eq!(
10481 first.header.ty,
10482 FrameType::HelloAck,
10483 "the first frame a registering module reads must be its HELLO_ACK"
10484 );
10485 assert_eq!(first.header.corr, 7);
10486 let second = module_rx.recv().await.unwrap().frame;
10487 assert_eq!(second.header.ty, FrameType::Request);
10488 assert!(
10489 matches!(
10490 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
10491 ModuleControlRequest::RouteBind { .. }
10492 ),
10493 "the route.bind follows the ack"
10494 );
10495 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
10496
10497 handler
10498 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
10499 .await
10500 .unwrap();
10501 assert!(open.await.unwrap().is_empty());
10502 let _ = client_rx.recv().await.unwrap();
10503 }
10504
10505 #[tokio::test(start_paused = true)]
10506 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
10507 let handler = ControlHandler::with_forwarding(
10508 Arc::new(Registry::default()),
10509 Arc::new(ForwardingTable::default()),
10510 );
10511 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
10512 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
10513 hello_via_sink(
10514 &handler,
10515 &first_ctx,
10516 &mut first_rx,
10517 hello_frame("first", PROTOCOL_VERSION, 1),
10518 )
10519 .await;
10520 hello_via_sink(
10521 &handler,
10522 &second_ctx,
10523 &mut second_rx,
10524 hello_frame("second", PROTOCOL_VERSION, 2),
10525 )
10526 .await;
10527 let root = unique_project_root("second-only");
10528 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
10529 let cloned = handler.clone();
10530 let open = tokio::spawn(async move {
10531 cloned
10532 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
10533 .await
10534 .unwrap()
10535 });
10536 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
10537 .await
10538 .unwrap()
10539 .unwrap();
10540 let first = query_live_roots(&handler, &first_ctx).await;
10541 let second = query_live_roots(&handler, &second_ctx).await;
10542 assert!(
10543 matches!(
10544 first,
10545 ModuleControlResponseToModule::LiveRoots {
10546 total_bindings: 0,
10547 ..
10548 }
10549 ),
10550 "cross-module scope must not expose another module's roots"
10551 );
10552 assert!(
10553 matches!(
10554 second,
10555 ModuleControlResponseToModule::LiveRoots {
10556 total_bindings: 1,
10557 ..
10558 }
10559 ),
10560 "second module must see its pending route"
10561 );
10562 handler
10563 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
10564 .await
10565 .unwrap();
10566 assert!(open.await.unwrap().is_empty());
10567 }
10568
10569 #[tokio::test(start_paused = true)]
10570 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
10571 let handler = ControlHandler::with_forwarding(
10572 Arc::new(Registry::default()),
10573 Arc::new(ForwardingTable::default()),
10574 );
10575 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
10576 hello_via_sink(
10577 &handler,
10578 &target_ctx,
10579 &mut target_rx,
10580 hello_frame("target", PROTOCOL_VERSION, 1),
10581 )
10582 .await;
10583 let actual = query_live_roots(&handler, &target_ctx).await;
10584 let ModuleControlResponseToModule::LiveRoots {
10585 roots,
10586 unknown_root_bindings,
10587 total_bindings,
10588 } = actual
10589 else {
10590 panic!("expected live roots")
10591 };
10592 assert!(roots.is_empty());
10593 assert_eq!(unknown_root_bindings, 0);
10594 assert_eq!(total_bindings, 0);
10595 assert_eq!(
10596 total_bindings,
10597 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
10598 );
10599 }
10600
10601 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
10608 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
10609 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
10610 let text = std::fs::read_to_string(&path)
10611 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
10612 let vectors: Vec<(String, Value)> = text
10613 .lines()
10614 .filter(|line| !line.trim().is_empty())
10615 .map(|line| {
10616 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
10617 let id = entry["corpus_id"]
10618 .as_str()
10619 .expect("every vector carries a corpus_id")
10620 .to_string();
10621 (id, entry["package"].clone())
10622 })
10623 .collect();
10624 assert_eq!(
10628 vectors.len(),
10629 3,
10630 "vendored fed corpus changed size; re-sync from subc-federation"
10631 );
10632
10633 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
10648 let richest = vectors
10649 .iter()
10650 .filter_map(|(_, package)| package.as_object())
10651 .map(|object| {
10652 object
10653 .keys()
10654 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
10655 .count()
10656 })
10657 .max()
10658 .unwrap_or(0);
10659 assert!(
10660 richest >= 2,
10661 "vendored corpus no longer carries a package with unmodelled fields, \
10662 so the relay test can no longer distinguish a verbatim relay from a lossy one"
10663 );
10664
10665 vectors
10666 }
10667
10668 #[tokio::test]
10676 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
10677 for (corpus_id, package) in fed_admission_facts_vectors() {
10678 let registry = Arc::new(Registry::default());
10679 let forwarding = Arc::new(ForwardingTable::default());
10680 let supervisor = SupervisorHandle::new();
10681 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10682 let handler =
10683 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10684 .with_supervisor(supervisor)
10685 .with_admission_facts_config(
10686 Some("fed".to_string()),
10687 Some(vec!["target".to_string()]),
10688 );
10689
10690 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
10691 hello_via_sink(
10692 &handler,
10693 &target_ctx,
10694 &mut target_rx,
10695 hello_frame("target", PROTOCOL_VERSION, 1),
10696 )
10697 .await;
10698
10699 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
10700 let route_handler = handler.clone();
10701 let expected = package.clone();
10702 let route_task = tokio::spawn(async move {
10703 route_handler
10704 .handle_control_frame(
10705 &client_ctx,
10706 route_open_frame_with_admission_facts(
10707 20,
10708 "target",
10709 unique_project_root("admission-facts"),
10710 Some(subc_control::ConsumerIdentity {
10711 module_id: "fed".to_string(),
10712 launch_nonce: "fed-nonce".to_string(),
10713 }),
10714 Some(package),
10715 ),
10716 )
10717 .await
10718 .unwrap()
10719 });
10720
10721 let bind_frame = target_rx.recv().await.unwrap();
10722 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10723 let ModuleControlRequest::RouteBind {
10724 admission_facts, ..
10725 } = bind
10726 else {
10727 panic!("{corpus_id}: expected route.bind")
10728 };
10729 assert_eq!(
10730 admission_facts,
10731 Some(expected),
10732 "{corpus_id}: relay must not add, drop or reshape any field"
10733 );
10734
10735 handler
10736 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10737 .await
10738 .unwrap();
10739 route_task.await.unwrap();
10740 }
10741 }
10742
10743 #[tokio::test]
10744 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
10745 let registry = Arc::new(Registry::default());
10746 let forwarding = Arc::new(ForwardingTable::default());
10747 let supervisor = SupervisorHandle::new();
10748 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
10749 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
10750 let handler =
10751 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
10752 .with_supervisor(supervisor)
10753 .with_admission_facts_config(
10754 Some("fed".to_string()),
10755 Some(vec!["target".to_string()]),
10756 );
10757
10758 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
10759 hello_via_sink(
10760 &handler,
10761 &target_ctx,
10762 &mut target_rx,
10763 hello_frame("target", PROTOCOL_VERSION, 1),
10764 )
10765 .await;
10766 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
10767 hello_via_sink(
10768 &handler,
10769 &other_ctx,
10770 &mut other_rx,
10771 hello_frame("other", PROTOCOL_VERSION, 2),
10772 )
10773 .await;
10774
10775 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
10776 let expected_facts = facts.clone();
10777 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
10778 let route_handler = handler.clone();
10779 let route_task = tokio::spawn(async move {
10780 route_handler
10781 .handle_control_frame(
10782 &client_ctx,
10783 route_open_frame_with_admission_facts(
10784 10,
10785 "target",
10786 unique_project_root("admission-facts"),
10787 Some(subc_control::ConsumerIdentity {
10788 module_id: "fed".to_string(),
10789 launch_nonce: "fed-nonce".to_string(),
10790 }),
10791 Some(facts.clone()),
10792 ),
10793 )
10794 .await
10795 .unwrap()
10796 });
10797 let bind_frame = target_rx.recv().await.unwrap();
10798 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10799 let ModuleControlRequest::RouteBind {
10800 admission_facts, ..
10801 } = bind
10802 else {
10803 panic!("expected route.bind")
10804 };
10805 assert_eq!(admission_facts, Some(expected_facts));
10806 handler
10807 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
10808 .await
10809 .unwrap();
10810 assert!(route_task.await.unwrap().is_empty());
10811 assert!(matches!(
10812 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
10813 .unwrap(),
10814 ClientControlResponse::RouteOpen { .. }
10815 ));
10816
10817 let direct = handler
10818 .handle_control_frame(
10819 &route_ctx(ConnectionId::new(73)).0,
10820 route_open_frame_with_admission_facts(
10821 11,
10822 "target",
10823 unique_project_root("admission-facts"),
10824 None,
10825 Some(json!({"x": 1})),
10826 ),
10827 )
10828 .await
10829 .unwrap();
10830 assert_eq!(
10831 parse_error(&direct[0])["code"],
10832 "admission_facts_not_permitted"
10833 );
10834
10835 let different_reserved = handler
10836 .handle_control_frame(
10837 &route_ctx(ConnectionId::new(77)).0,
10838 route_open_frame_with_admission_facts(
10839 15,
10840 "target",
10841 unique_project_root("admission-facts"),
10842 Some(subc_control::ConsumerIdentity {
10843 module_id: "other".to_string(),
10844 launch_nonce: "other-nonce".to_string(),
10845 }),
10846 Some(json!({"x": 1})),
10847 ),
10848 )
10849 .await
10850 .unwrap();
10851 assert_eq!(
10852 parse_error(&different_reserved[0])["code"],
10853 "admission_facts_not_permitted"
10854 );
10855
10856 let other_target = handler
10857 .handle_control_frame(
10858 &route_ctx(ConnectionId::new(74)).0,
10859 route_open_frame_with_admission_facts(
10860 12,
10861 "other",
10862 unique_project_root("admission-facts"),
10863 Some(subc_control::ConsumerIdentity {
10864 module_id: "fed".to_string(),
10865 launch_nonce: "fed-nonce".to_string(),
10866 }),
10867 Some(json!({"x": 1})),
10868 ),
10869 )
10870 .await
10871 .unwrap();
10872 assert_eq!(
10873 parse_error(&other_target[0])["code"],
10874 "admission_facts_target_not_allowed"
10875 );
10876
10877 let nonexistent = handler
10878 .handle_control_frame(
10879 &route_ctx(ConnectionId::new(75)).0,
10880 route_open_frame_with_admission_facts(
10881 13,
10882 "missing",
10883 unique_project_root("admission-facts"),
10884 None,
10885 Some(json!({"x": 1})),
10886 ),
10887 )
10888 .await
10889 .unwrap();
10890 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
10891
10892 let described = handler
10893 .handle_control_frame(
10894 &route_ctx(ConnectionId::new(76)).0,
10895 Frame::build(
10896 FrameType::Request,
10897 control_flags(),
10898 0,
10899 0,
10900 14,
10901 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10902 )
10903 .unwrap(),
10904 )
10905 .await
10906 .unwrap();
10907 let ClientControlResponse::ServerDescribe { capabilities, .. } =
10908 serde_json::from_slice(&described[0].body).unwrap()
10909 else {
10910 panic!("expected server.describe response")
10911 };
10912 assert!(capabilities
10913 .iter()
10914 .any(|cap| cap == "admission_facts_relay_v1"));
10915 }
10916
10917 #[tokio::test]
10918 async fn admission_facts_without_configured_carrier_are_rejected() {
10919 let registry = Arc::new(Registry::default());
10920 let forwarding = Arc::new(ForwardingTable::default());
10921 let handler = ControlHandler::with_forwarding(registry, forwarding);
10922 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
10923 hello_via_sink(
10924 &handler,
10925 &target_ctx,
10926 &mut target_rx,
10927 hello_frame("target", PROTOCOL_VERSION, 1),
10928 )
10929 .await;
10930
10931 let responses = handler
10932 .handle_control_frame(
10933 &route_ctx(ConnectionId::new(79)).0,
10934 route_open_frame_with_admission_facts(
10935 16,
10936 "target",
10937 unique_project_root("admission-facts"),
10938 None,
10939 Some(json!({"x": 1})),
10940 ),
10941 )
10942 .await
10943 .unwrap();
10944 assert_eq!(
10945 parse_error(&responses[0])["code"],
10946 "admission_facts_not_permitted"
10947 );
10948 }
10949
10950 #[tokio::test]
10951 async fn route_open_relays_consumer_capabilities_verbatim() {
10952 let registry = Arc::new(Registry::default());
10953 let forwarding = Arc::new(ForwardingTable::default());
10954 let handler =
10955 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10956 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
10957 hello_via_sink(
10958 &handler,
10959 &module_ctx,
10960 &mut module_rx,
10961 hello_frame("aft", PROTOCOL_VERSION, 7),
10962 )
10963 .await;
10964
10965 let expected = vec!["elicitation".to_string(), "roots".to_string()];
10966 let expected_for_request = expected.clone();
10967 let project_root = unique_project_root("consumer-capabilities-present");
10968 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
10969 let route_handler = handler.clone();
10970 let route_task = tokio::spawn(async move {
10971 route_handler
10972 .handle_control_frame(
10973 &client_ctx,
10974 route_open_frame_with_consumer_capabilities(
10975 401,
10976 "aft",
10977 project_root,
10978 Some(expected_for_request),
10979 ),
10980 )
10981 .await
10982 .unwrap()
10983 });
10984 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10985 .await
10986 .unwrap()
10987 .unwrap();
10988 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
10989 let ModuleControlRequest::RouteBind {
10990 consumer_capabilities,
10991 ..
10992 } = bind
10993 else {
10994 panic!("expected route.bind request, got {bind:?}");
10995 };
10996 assert_eq!(consumer_capabilities, Some(expected.clone()));
10997
10998 handler
10999 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11000 .await
11001 .unwrap();
11002 let route_response = route_task.await.unwrap();
11003 assert!(route_response.is_empty());
11004 let published = client_rx.recv().await.unwrap();
11005 assert!(matches!(
11006 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11007 ClientControlResponse::RouteOpen { .. }
11008 ));
11009 }
11010
11011 #[tokio::test]
11012 async fn route_open_without_consumer_capabilities_relays_none() {
11013 let registry = Arc::new(Registry::default());
11014 let forwarding = Arc::new(ForwardingTable::default());
11015 let handler =
11016 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11017 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
11018 hello_via_sink(
11019 &handler,
11020 &module_ctx,
11021 &mut module_rx,
11022 hello_frame("aft", PROTOCOL_VERSION, 7),
11023 )
11024 .await;
11025
11026 let project_root = unique_project_root("consumer-capabilities-absent");
11027 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
11028 let route_handler = handler.clone();
11029 let route_task = tokio::spawn(async move {
11030 route_handler
11031 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
11032 .await
11033 .unwrap()
11034 });
11035 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11036 .await
11037 .unwrap()
11038 .unwrap();
11039 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11040 let ModuleControlRequest::RouteBind {
11041 consumer_capabilities,
11042 ..
11043 } = bind
11044 else {
11045 panic!("expected route.bind request, got {bind:?}");
11046 };
11047 assert_eq!(consumer_capabilities, None);
11048
11049 handler
11050 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11051 .await
11052 .unwrap();
11053 let route_response = route_task.await.unwrap();
11054 assert!(route_response.is_empty());
11055 let published = client_rx.recv().await.unwrap();
11056 assert!(matches!(
11057 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11058 ClientControlResponse::RouteOpen { .. }
11059 ));
11060 }
11061
11062 async fn bind_role_versions_for(
11066 sent: Option<BTreeMap<String, String>>,
11067 connection: u64,
11068 ) -> Option<BTreeMap<String, String>> {
11069 let registry = Arc::new(Registry::default());
11070 let forwarding = Arc::new(ForwardingTable::default());
11071 let handler =
11072 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11073 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
11074 hello_via_sink(
11075 &handler,
11076 &module_ctx,
11077 &mut module_rx,
11078 hello_frame("aft", PROTOCOL_VERSION, 7),
11079 )
11080 .await;
11081 let project_root = unique_project_root("role-versions");
11082 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
11083 let route_handler = handler.clone();
11084 let route_task = tokio::spawn(async move {
11085 route_handler
11086 .handle_control_frame(
11087 &client_ctx,
11088 route_open_frame_with_role_versions(403, "aft", project_root, sent),
11089 )
11090 .await
11091 .unwrap()
11092 });
11093 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11094 .await
11095 .expect("a well-formed route.open reaches the module as a bind")
11096 .unwrap();
11097 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11098 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
11099 panic!("expected route.bind request, got {bind:?}");
11100 };
11101 handler
11102 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11103 .await
11104 .unwrap();
11105 assert!(route_task.await.unwrap().is_empty());
11106 let published = client_rx.recv().await.unwrap();
11107 assert!(matches!(
11108 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11109 ClientControlResponse::RouteOpen { .. }
11110 ));
11111 role_versions
11112 }
11113
11114 #[tokio::test]
11115 async fn route_open_relays_role_versions_verbatim() {
11116 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
11117 assert_eq!(
11118 bind_role_versions_for(Some(sent.clone()), 141).await,
11119 Some(sent)
11120 );
11121 }
11122
11123 #[tokio::test]
11126 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
11127 assert_eq!(bind_role_versions_for(None, 143).await, None);
11128 assert_eq!(
11129 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
11130 None
11131 );
11132 }
11133
11134 #[tokio::test]
11135 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
11136 let registry = Arc::new(Registry::default());
11137 let forwarding = Arc::new(ForwardingTable::default());
11138 let handler =
11139 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11140 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
11141 hello_via_sink(
11142 &handler,
11143 &module_ctx,
11144 &mut module_rx,
11145 hello_frame("aft", PROTOCOL_VERSION, 7),
11146 )
11147 .await;
11148
11149 let nine: BTreeMap<String, String> = (0..9)
11150 .map(|index| (format!("role-{index}"), "v1".to_string()))
11151 .collect();
11152 for (label, malformed) in [
11153 (
11154 "invalid role name",
11155 role_versions(&[("Tool_Provider", "v1")]),
11156 ),
11157 ("invalid version", role_versions(&[("tool-provider", "v0")])),
11158 ("nine entries", nine),
11159 ] {
11160 let responses = tokio::time::timeout(
11163 Duration::from_secs(1),
11164 handler.handle_control_frame(
11165 &route_ctx(ConnectionId::new(148)).0,
11166 route_open_frame_with_role_versions(
11167 404,
11168 "aft",
11169 unique_project_root("role-versions-malformed"),
11170 Some(malformed),
11171 ),
11172 ),
11173 )
11174 .await
11175 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
11176 .unwrap();
11177 assert_eq!(responses.len(), 1, "{label}");
11178 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
11179 let error = parse_error(&responses[0]);
11180 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
11181 assert_eq!(
11182 error["detail"]["field"], "role_versions",
11183 "{label}: {error}"
11184 );
11185 assert!(
11186 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
11187 "{label}: a malformed declaration is terminal"
11188 );
11189 assert!(
11190 module_rx.try_recv().is_err(),
11191 "{label}: the module must never see a bind"
11192 );
11193 }
11194 }
11195
11196 #[tokio::test]
11200 async fn route_role_versions_capability_is_advertised() {
11201 let handler = ControlHandler::new(Arc::new(Registry::default()));
11202 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
11203 let ack = hello_via_sink(
11204 &handler,
11205 &ctx,
11206 &mut rx,
11207 hello_frame("m", PROTOCOL_VERSION, 1),
11208 )
11209 .await;
11210 let ack = parse_ack(&ack);
11211 assert!(
11212 ack.subc_capabilities
11213 .iter()
11214 .any(|c| c == "route-role-versions/v1"),
11215 "{:?}",
11216 ack.subc_capabilities
11217 );
11218
11219 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11220 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11221 let reply = handler
11222 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
11223 .await
11224 .unwrap()
11225 .pop()
11226 .unwrap();
11227 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11228 serde_json::from_slice(&reply.body).unwrap()
11229 else {
11230 panic!("not a server.describe reply");
11231 };
11232 assert!(
11233 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
11234 "{capabilities:?}"
11235 );
11236 }
11237
11238 #[tokio::test]
11242 async fn catalog_events_capability_is_advertised() {
11243 let handler = ControlHandler::new(Arc::new(Registry::default()));
11244 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11245 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11246 let reply = handler
11247 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
11248 .await
11249 .unwrap()
11250 .pop()
11251 .unwrap();
11252 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11253 serde_json::from_slice(&reply.body).unwrap()
11254 else {
11255 panic!("not a server.describe reply");
11256 };
11257 assert!(
11258 capabilities
11259 .iter()
11260 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
11261 "{capabilities:?}"
11262 );
11263 }
11264
11265 #[tokio::test]
11266 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
11267 let handler = ControlHandler::new(Arc::new(Registry::default()));
11268 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
11269 let ack = hello_via_sink(
11270 &handler,
11271 &module,
11272 &mut module_rx,
11273 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
11274 )
11275 .await;
11276 let ack = parse_ack(&ack);
11277 assert!(
11278 ack.subc_capabilities
11279 .iter()
11280 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11281 "{:?}",
11282 ack.subc_capabilities
11283 );
11284 assert!(ack
11285 .subc_capabilities
11286 .iter()
11287 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11288
11289 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
11290 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11291 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11292 let reply = handler
11293 .handle_control_frame(&client, frame)
11294 .await
11295 .unwrap()
11296 .pop()
11297 .unwrap();
11298 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11299 serde_json::from_slice(&reply.body).unwrap()
11300 else {
11301 panic!("not a server.describe reply");
11302 };
11303 assert!(
11304 capabilities
11305 .iter()
11306 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
11307 "{capabilities:?}"
11308 );
11309 assert!(capabilities
11310 .iter()
11311 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
11312 }
11313
11314 #[tokio::test]
11315 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
11316 let handler = ControlHandler::new(Arc::new(Registry::default()));
11317 let (module, mut rx) = route_ctx(ConnectionId::new(154));
11318 let ack = hello_via_sink(
11319 &handler,
11320 &module,
11321 &mut rx,
11322 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
11323 )
11324 .await;
11325 assert!(parse_ack(&ack)
11326 .subc_capabilities
11327 .iter()
11328 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11329 let (client, _rx) = route_ctx(ConnectionId::new(155));
11330 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
11331 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
11332 let reply = handler
11333 .handle_control_frame(&client, frame)
11334 .await
11335 .unwrap()
11336 .pop()
11337 .unwrap();
11338 let ClientControlResponse::ServerDescribe { capabilities, .. } =
11339 serde_json::from_slice(&reply.body).unwrap()
11340 else {
11341 panic!("not a server.describe reply")
11342 };
11343 assert!(capabilities
11344 .iter()
11345 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
11346 }
11347
11348 #[tokio::test]
11349 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
11350 let registry = Arc::new(Registry::default());
11351 let forwarding = Arc::new(ForwardingTable::default());
11352 let handler =
11353 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11354 .with_health_probe_timeout(Duration::from_secs(5));
11355 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
11356 hello_via_sink(
11357 &handler,
11358 &module_ctx,
11359 &mut module_rx,
11360 non_routable_hello_frame_with_control_ops(
11361 "mcp",
11362 300,
11363 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11364 ),
11365 )
11366 .await;
11367 assert!(registry
11368 .get_module("mcp")
11369 .unwrap()
11370 .unwrap()
11371 .manifest
11372 .provides
11373 .is_empty());
11374
11375 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
11376 let route_response = handler
11377 .handle_control_frame(
11378 &route_client_ctx,
11379 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
11380 )
11381 .await
11382 .unwrap();
11383 assert_eq!(route_response[0].header.ty, FrameType::Error);
11384 assert_eq!(
11385 parse_error(&route_response[0])["code"],
11386 "target_unavailable"
11387 );
11388 assert!(parse_error(&route_response[0])["message"]
11389 .as_str()
11390 .unwrap()
11391 .contains("does not provide the requested target"));
11392 assert!(module_rx.try_recv().is_err());
11393
11394 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
11395 let health_handler = handler.clone();
11396 let health_task = tokio::spawn(async move {
11397 health_handler
11398 .handle_control_frame(
11399 &health_client_ctx,
11400 supervisor_health_probe_frame(302, "mcp"),
11401 )
11402 .await
11403 .unwrap()
11404 });
11405 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11406 .await
11407 .unwrap()
11408 .unwrap();
11409 assert_eq!(
11410 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11411 ModuleControlRequest::HealthCheck {}
11412 );
11413 handler
11414 .handle_control_frame(
11415 &module_ctx,
11416 health_response(health_frame.header.corr, HealthStatus::Ok),
11417 )
11418 .await
11419 .unwrap();
11420 let health_response = health_task.await.unwrap();
11421 assert_eq!(health_response[0].header.ty, FrameType::Response);
11422 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11423 ClientControlResponse::SupervisorHealthProbe {
11424 module_id, status, ..
11425 } => {
11426 assert_eq!(module_id, "mcp");
11427 assert_eq!(status, HealthStatus::Ok);
11428 }
11429 other => panic!("unexpected health response: {other:?}"),
11430 }
11431
11432 forwarding
11437 .cleanup_connection(module_ctx.connection_id)
11438 .unwrap();
11439 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
11440 let cleanup_response = tokio::time::timeout(
11441 Duration::from_millis(200),
11442 handler.handle_control_frame(
11443 &cleanup_probe_ctx,
11444 supervisor_health_probe_frame(303, "mcp"),
11445 ),
11446 )
11447 .await
11448 .expect("probe should fail immediately when the control lane is gone")
11449 .unwrap();
11450 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
11451 assert_eq!(
11452 parse_error(&cleanup_response[0])["code"],
11453 "target_unavailable"
11454 );
11455 assert!(parse_error(&cleanup_response[0])["message"]
11456 .as_str()
11457 .unwrap()
11458 .contains("no module connection"));
11459
11460 handler
11461 .cleanup_connection(module_ctx.connection_id)
11462 .unwrap();
11463 }
11464
11465 #[tokio::test]
11466 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
11467 let registry = Arc::new(Registry::default());
11468 let supervisor_handle = SupervisorHandle::new();
11469 let supervisor =
11470 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11471 .with_handle(supervisor_handle.clone())
11472 .with_connection_file_path(
11473 std::env::temp_dir()
11474 .join(format!("subc-route-open-warming-{}", std::process::id())),
11475 );
11476 let module = supervisor
11477 .supervise_configured(
11478 ModuleSpec {
11479 module_id: "warming".to_string(),
11480 program: fake_aft_stub_path(),
11481 args: Vec::new(),
11482 env: Vec::new(),
11483 reserved: false,
11484 reserved_prefixes: Vec::new(),
11485 protocol: ModuleProtocol::Subc,
11486 overlap: Default::default(),
11487 },
11488 true,
11489 )
11490 .unwrap();
11491 assert_eq!(module.state().unwrap(), ModuleState::Running);
11492
11493 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11494 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
11495 let response = handler
11496 .handle_control_frame(
11497 &ctx,
11498 route_open_frame(304, "warming", unique_project_root("warming")),
11499 )
11500 .await
11501 .unwrap();
11502 module.stop().await.unwrap();
11503
11504 assert_eq!(response[0].header.ty, FrameType::Error);
11505 let error = parse_error(&response[0]);
11506 assert_eq!(error["code"], "module_warming");
11507 assert!(error["message"]
11508 .as_str()
11509 .unwrap()
11510 .contains("state=running, enabled=true, live=false"));
11511 }
11512
11513 #[test]
11514 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
11515 let handler = ControlHandler::new(Arc::new(Registry::default()));
11516 let capture = EventCapture::default();
11517 let _subscriber =
11518 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11519 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
11520 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
11521 let pending = (0..limit).collect::<Vec<_>>();
11522 let response = handler
11523 .route_open_capacity_refusal(
11524 &ctx,
11525 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
11526 "busy",
11527 pending.len(),
11528 limit,
11529 )
11530 .unwrap();
11531 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11532 let event = capture
11533 .events()
11534 .into_iter()
11535 .find(|event| {
11536 event.target == "control"
11537 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
11538 })
11539 .expect("connection admission refusal event");
11540 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11541 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11542 }
11543
11544 #[test]
11545 fn route_open_target_cap_logs_admission_reason_and_capacity() {
11546 let handler = ControlHandler::new(Arc::new(Registry::default()));
11547 let capture = EventCapture::default();
11548 let _subscriber =
11549 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11550 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
11551 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
11552 let guards = (0..limit)
11553 .map(|_| {
11554 handler
11555 .route_bind_concurrency
11556 .try_admit("busy", limit)
11557 .unwrap()
11558 })
11559 .collect::<Vec<_>>();
11560 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
11561 Err(in_flight) => in_flight,
11562 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
11563 };
11564 let response = handler
11565 .route_open_target_capacity_refusal(
11566 &ctx,
11567 &route_open_frame(397, "busy", unique_project_root("target-cap")),
11568 "busy",
11569 in_flight,
11570 )
11571 .unwrap();
11572 assert_eq!(parse_error(&response)["code"], "target_unavailable");
11573 let event = capture
11574 .events()
11575 .into_iter()
11576 .find(|event| {
11577 event.target == "control"
11578 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
11579 })
11580 .expect("target admission refusal event");
11581 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
11582 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
11583 drop(guards);
11584 }
11585
11586 #[tokio::test(flavor = "current_thread")]
11587 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
11588 let handler = ControlHandler::new(Arc::new(Registry::default()));
11589 let capture = EventCapture::default();
11590 let _subscriber =
11591 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11592 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
11593 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11594
11595 let _ = handler
11596 .handle_client_control_request(
11597 &ctx,
11598 frame,
11599 ClientControlRequest::SupervisorSetEnabled {
11600 module_id: "broca".to_string(),
11601 enabled: false,
11602 },
11603 &mut None,
11604 )
11605 .await
11606 .unwrap();
11607
11608 let events = capture
11609 .events()
11610 .into_iter()
11611 .filter(|event| {
11612 event.target == "control"
11613 && event.fields.get("message").map(String::as_str)
11614 == Some("supervisor request received")
11615 })
11616 .collect::<Vec<_>>();
11617 assert_eq!(events.len(), 1, "one supervisor request log line");
11618 let event = &events[0];
11619 assert_eq!(event.level, tracing::Level::INFO);
11620 assert_eq!(
11621 event.fields.get("op"),
11622 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
11623 );
11624 assert_eq!(
11625 event.fields.get("module_id"),
11626 Some(&"\"broca\"".to_string())
11627 );
11628 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
11629 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
11630 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
11631 }
11632
11633 #[tokio::test(flavor = "current_thread")]
11634 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
11635 let registry = Arc::new(Registry::default());
11636 let module_connection = ConnectionId::new(303);
11637 registry
11638 .register_with_control_ops(
11639 manifest("aft", PROTOCOL_VERSION),
11640 PROTOCOL_VERSION,
11641 module_connection,
11642 module_baseline_control_ops(),
11643 )
11644 .unwrap();
11645 let handler = ControlHandler::new(registry);
11646 let capture = EventCapture::default();
11647 let _subscriber =
11648 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11649 let (ctx, _rx) = route_ctx(module_connection);
11650 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
11651
11652 let _ = handler
11653 .handle_client_control_request(
11654 &ctx,
11655 frame,
11656 ClientControlRequest::SupervisorSetEnabled {
11657 module_id: "broca".to_string(),
11658 enabled: false,
11659 },
11660 &mut None,
11661 )
11662 .await
11663 .unwrap();
11664
11665 let callers = capture
11666 .events()
11667 .into_iter()
11668 .filter(|event| {
11669 event.fields.get("message").map(String::as_str)
11670 == Some("supervisor request received")
11671 })
11672 .map(|event| event.fields.get("caller").cloned())
11673 .collect::<Vec<_>>();
11674 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
11675 }
11676
11677 #[tokio::test(flavor = "current_thread")]
11678 async fn supervisor_list_does_not_log_request_received() {
11679 let handler = ControlHandler::new(Arc::new(Registry::default()));
11680 let capture = EventCapture::default();
11681 let _subscriber =
11682 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11683 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
11684 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
11685
11686 handler
11687 .handle_client_control_request(
11688 &ctx,
11689 frame,
11690 ClientControlRequest::SupervisorList {},
11691 &mut None,
11692 )
11693 .await
11694 .unwrap();
11695
11696 assert!(capture.events().into_iter().all(|event| {
11697 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11698 }));
11699 }
11700
11701 #[tokio::test(flavor = "current_thread")]
11702 async fn supervisor_rescan_preview_does_not_log_request_received() {
11703 let handler = ControlHandler::new(Arc::new(Registry::default()));
11704 let capture = EventCapture::default();
11705 let _subscriber =
11706 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11707 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
11708 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
11709
11710 handler
11711 .handle_client_control_request(
11712 &ctx,
11713 frame,
11714 ClientControlRequest::SupervisorRescan { preview: true },
11715 &mut None,
11716 )
11717 .await
11718 .unwrap();
11719
11720 assert!(capture.events().into_iter().all(|event| {
11721 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
11722 }));
11723 }
11724
11725 struct ApproveOperator;
11726 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
11727 fn prompt(
11728 &self,
11729 _: &str,
11730 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
11731 ) -> crate::operator_confirm::ProviderResult {
11732 crate::operator_confirm::ProviderResult::Approved
11733 }
11734 }
11735
11736 #[tokio::test]
11737 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
11738 assert_eq!(
11739 ControlHandler::default().privileged_operator_prompt,
11740 PrivilegedOperatorPrompt::Log
11741 );
11742 for (initial, changed, running) in [
11743 ("log", "enforce", PrivilegedOperatorPrompt::Log),
11744 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
11745 ] {
11746 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
11747 let path = dir.join("subc.jsonc");
11748 std::fs::write(
11749 &path,
11750 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
11751 )
11752 .unwrap();
11753 let config = crate::daemon_config::load(&path).unwrap().unwrap();
11754 let registry = Arc::new(Registry::default());
11755 let handle = SupervisorHandle::new();
11756 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
11757 .with_handle(handle.clone());
11758 let handler = ControlHandler::new(registry)
11759 .with_supervisor(handle)
11760 .with_privileged_operator_prompt(config.privileged_operator_prompt)
11761 .with_supervisor_rescan(supervisor, &path, config.port);
11762 handler.forwarding.operator_confirms().configure(
11763 Arc::new(ApproveOperator),
11764 crate::operator_confirm::OperatorLimits::default(),
11765 );
11766 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
11767
11768 for (disk_mode, needs_restart) in [
11769 (Some(initial), false),
11770 (Some(changed), true),
11771 (Some(changed), true),
11772 (None, running == PrivilegedOperatorPrompt::Enforce),
11773 (Some(initial), false),
11774 ] {
11775 let mut doc = json!({"version":1});
11776 if let Some(mode) = disk_mode {
11777 doc["privileged_operator_prompt"] = json!(mode);
11778 }
11779 std::fs::write(&path, doc.to_string()).unwrap();
11780 for preview in [true, false] {
11781 let request = ClientControlRequest::SupervisorRescan { preview };
11782 let frame = Frame::build(
11783 FrameType::Request,
11784 control_flags(),
11785 0,
11786 0,
11787 1,
11788 serde_json::to_vec(&request).unwrap(),
11789 )
11790 .unwrap();
11791 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11792 assert_eq!(frames.len(), 1);
11793 let ClientControlResponse::SupervisorRescan { result } =
11794 serde_json::from_slice(&frames[0].body).unwrap()
11795 else {
11796 panic!("expected a rescan response");
11797 };
11798 let expected = if needs_restart {
11799 vec!["privileged_operator_prompt"]
11800 } else {
11801 Vec::new()
11802 };
11803 assert_eq!(result.restart_required, expected);
11804 assert_eq!(handler.privileged_operator_prompt, running);
11805 assert_eq!(
11806 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
11807 running
11808 );
11809 }
11810 }
11811 }
11812 }
11813
11814 #[tokio::test]
11815 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
11816 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
11817 let path = dir.join("subc.jsonc");
11818 let registry = Arc::new(Registry::default());
11819 let handle = SupervisorHandle::new();
11820 let supervisor =
11821 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
11822 let handler = ControlHandler::new(registry)
11823 .with_supervisor(handle)
11824 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
11825 .with_supervisor_rescan(supervisor, &path, None);
11826 handler.forwarding.operator_confirms().configure(
11827 Arc::new(ApproveOperator),
11828 crate::operator_confirm::OperatorLimits::default(),
11829 );
11830 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
11831 for preview in [true, false] {
11832 std::fs::write(
11833 &path,
11834 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
11835 )
11836 .unwrap();
11837 let request = ClientControlRequest::SupervisorRescan { preview };
11838 let frame = Frame::build(
11839 FrameType::Request,
11840 control_flags(),
11841 0,
11842 0,
11843 1,
11844 serde_json::to_vec(&request).unwrap(),
11845 )
11846 .unwrap();
11847 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
11848 assert_eq!(frames.len(), 1);
11849 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
11850 assert_eq!(
11851 handler.privileged_operator_prompt,
11852 PrivilegedOperatorPrompt::Enforce
11853 );
11854 }
11855 }
11856
11857 #[tokio::test]
11862 async fn route_open_refusal_names_the_check_that_refused() {
11863 let handler = ControlHandler::new(Arc::new(Registry::default()));
11864 let capture = EventCapture::default();
11865 let _subscriber =
11866 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11867 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
11868 let response = handler
11869 .handle_control_frame(
11870 &ctx,
11871 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
11872 )
11873 .await
11874 .unwrap();
11875
11876 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
11877 let event = capture
11878 .events()
11879 .into_iter()
11880 .find(|event| {
11881 event.target == "control"
11882 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
11883 })
11884 .expect("route.open refusal event");
11885 assert_eq!(
11886 event.fields.get("reason"),
11887 Some(&"\"not_registered\"".to_string())
11888 );
11889 }
11890
11891 #[tokio::test]
11892 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
11893 let registry = Arc::new(Registry::default());
11894 let supervisor_handle = SupervisorHandle::new();
11895 let supervisor =
11896 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
11897 .with_handle(supervisor_handle.clone())
11898 .with_connection_file_path(std::env::temp_dir().join(format!(
11899 "subc-route-open-refusal-info-{}",
11900 std::process::id()
11901 )));
11902 let module = supervisor
11903 .supervise_configured(
11904 ModuleSpec {
11905 module_id: "warming".to_string(),
11906 program: fake_aft_stub_path(),
11907 args: Vec::new(),
11908 env: Vec::new(),
11909 reserved: false,
11910 reserved_prefixes: Vec::new(),
11911 protocol: ModuleProtocol::Subc,
11912 overlap: Default::default(),
11913 },
11914 true,
11915 )
11916 .unwrap();
11917 assert_eq!(module.state().unwrap(), ModuleState::Running);
11918
11919 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
11920 assert!(handler
11921 .counters()
11922 .snapshot()
11923 .get("route_open_refused_by_code")
11924 .is_none());
11925 let capture = EventCapture::default();
11926 let _subscriber =
11927 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11928 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
11929 let response = handler
11930 .handle_control_frame(
11931 &ctx,
11932 route_open_frame(394, "warming", unique_project_root("refusal-info")),
11933 )
11934 .await
11935 .unwrap();
11936 module.stop().await.unwrap();
11937
11938 assert_eq!(parse_error(&response[0])["code"], "module_warming");
11939 let event = capture
11940 .events()
11941 .into_iter()
11942 .find(|event| {
11943 event.target == "control"
11944 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
11945 })
11946 .expect("route.open refusal event");
11947 assert_eq!(
11948 event.fields.get("module_id"),
11949 Some(&"\"warming\"".to_string())
11950 );
11951 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
11952 assert_eq!(
11953 event.fields.get("reason"),
11954 Some(&"\"supervised_not_registered\"".to_string())
11955 );
11956 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
11957 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
11958 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
11959 assert_eq!(
11960 handler.counters().snapshot()["route_open_refused_by_code"],
11961 json!({ "module_warming": 1 })
11962 );
11963 }
11964
11965 const OUTAGE_START: &str = "route.open refusing module: not serving";
11966 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
11967
11968 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
11969 capture
11970 .events()
11971 .into_iter()
11972 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
11973 .collect()
11974 }
11975
11976 fn supervise_stub(
11977 registry: &Arc<Registry>,
11978 module_id: &str,
11979 enabled: bool,
11980 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
11981 let supervisor_handle = SupervisorHandle::new();
11982 let supervisor =
11983 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
11984 .with_handle(supervisor_handle.clone())
11985 .with_connection_file_path(std::env::temp_dir().join(format!(
11986 "subc-route-outage-{module_id}-{}",
11987 std::process::id()
11988 )));
11989 let module = supervisor
11990 .supervise_configured(
11991 ModuleSpec {
11992 module_id: module_id.to_string(),
11993 program: fake_aft_stub_path(),
11994 args: Vec::new(),
11995 env: Vec::new(),
11996 reserved: false,
11997 reserved_prefixes: Vec::new(),
11998 protocol: ModuleProtocol::Subc,
11999 overlap: Default::default(),
12000 },
12001 enabled,
12002 )
12003 .unwrap();
12004 (supervisor_handle, module)
12005 }
12006
12007 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
12008 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
12009 module_id: module_id.to_string(),
12010 drain_timeout_ms: Some(50),
12011 })
12012 .unwrap();
12013 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
12014 }
12015
12016 #[test]
12020 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
12021 let registry = Arc::new(Registry::default());
12022 let forwarding = Arc::new(ForwardingTable::default());
12023 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12024 let second = ControlHandler::with_forwarding(registry, forwarding);
12025 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
12026 }
12027
12028 #[tokio::test(flavor = "current_thread")]
12032 async fn route_open_unknown_module_refusals_add_no_outage_state() {
12033 let handler = ControlHandler::new(Arc::new(Registry::default()));
12034 let capture = EventCapture::default();
12035 let _subscriber =
12036 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12037 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
12038 for corr in 0..8 {
12039 let response = handler
12040 .handle_control_frame(
12041 &ctx,
12042 route_open_frame(
12043 380 + corr,
12044 &format!("nobody-{corr}"),
12045 unique_project_root("outage-unknown"),
12046 ),
12047 )
12048 .await
12049 .unwrap();
12050 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12051 }
12052
12053 assert_eq!(handler.route_outages.tracked_module_count(), 0);
12054 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
12055 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
12056 }
12057
12058 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12063 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
12064 let registry = Arc::new(Registry::default());
12065 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
12066 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12067 let capture = EventCapture::default();
12068 let _subscriber =
12069 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12070 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
12071 handler.route_outages.record_accepted("outage-restart");
12074
12075 let response = handler
12076 .handle_control_frame(
12077 &ctx,
12078 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
12079 )
12080 .await
12081 .unwrap();
12082 assert_eq!(response[0].header.ty, FrameType::Error);
12083 let starts = outage_lines(&capture, OUTAGE_START);
12084 assert_eq!(starts.len(), 1, "{starts:?}");
12085 assert_eq!(starts[0].level, tracing::Level::WARN);
12086 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
12087 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
12088 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
12089 handler.route_outages.record_accepted("outage-restart");
12090 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
12091
12092 let restart = handler
12093 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
12094 .await
12095 .unwrap();
12096 assert_eq!(
12097 restart[0].header.ty,
12098 FrameType::Response,
12099 "{:?}",
12100 parse_error(&restart[0])
12101 );
12102 handler
12103 .handle_control_frame(
12104 &ctx,
12105 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
12106 )
12107 .await
12108 .unwrap();
12109 module.stop().await.unwrap();
12110
12111 let starts = outage_lines(&capture, OUTAGE_START);
12112 assert_eq!(starts.len(), 2, "{starts:?}");
12113 assert_eq!(starts[1].level, tracing::Level::INFO);
12114 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
12115 }
12116
12117 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
12121 async fn failed_operator_restart_leaves_no_operator_mark() {
12122 let registry = Arc::new(Registry::default());
12123 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
12124 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12125 let capture = EventCapture::default();
12126 let _subscriber =
12127 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12128 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
12129 handler.route_outages.record_accepted("outage-disabled");
12130
12131 let restart = handler
12132 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
12133 .await
12134 .unwrap();
12135 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
12136 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
12137
12138 handler
12139 .handle_control_frame(
12140 &ctx,
12141 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
12142 )
12143 .await
12144 .unwrap();
12145 let starts = outage_lines(&capture, OUTAGE_START);
12146 assert_eq!(starts.len(), 1, "{starts:?}");
12147 assert_eq!(starts[0].level, tracing::Level::WARN);
12148 }
12149
12150 #[tokio::test(flavor = "current_thread")]
12151 async fn route_open_unknown_module_escapes_target_module_id() {
12152 let handler = ControlHandler::new(Arc::new(Registry::default()));
12153 let capture = EventCapture::default();
12154 let _subscriber =
12155 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12156 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
12157 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
12158 let response = handler
12159 .handle_control_frame(
12160 &ctx,
12161 route_open_frame(
12162 395,
12163 hostile_module_id,
12164 unique_project_root("hostile-target-module-id"),
12165 ),
12166 )
12167 .await
12168 .unwrap();
12169
12170 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
12171 let event = capture
12172 .events()
12173 .into_iter()
12174 .find(|event| {
12175 event.target == "control"
12176 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
12177 })
12178 .expect("route.open unknown-module refusal event");
12179 let logged = event.fields.get("module_id").expect("module_id field");
12180 assert!(!logged.bytes().any(|byte| byte < 0x20));
12181 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12182 }
12183
12184 #[tokio::test(flavor = "current_thread")]
12185 async fn route_open_module_rejection_uses_daemon_counter_key() {
12186 let registry = Arc::new(Registry::default());
12187 let forwarding = Arc::new(ForwardingTable::default());
12188 let handler =
12189 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12190 let module_connection = ConnectionId::new(95);
12191 let (module_ctx, mut module_rx) = route_ctx(module_connection);
12192 hello_via_sink(
12193 &handler,
12194 &module_ctx,
12195 &mut module_rx,
12196 hello_frame("aft", PROTOCOL_VERSION, 395),
12197 )
12198 .await;
12199
12200 let client_connection = ConnectionId::new(96);
12201 let (client_ctx, _client_rx) = route_ctx(client_connection);
12202 let capture = EventCapture::default();
12203 let _subscriber =
12204 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
12205 let (route_task, bind) = relay_route_open(
12206 &handler,
12207 client_connection,
12208 &client_ctx.egress,
12209 &mut module_rx,
12210 396,
12211 "aft",
12212 "hostile-module-code",
12213 )
12214 .await;
12215 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
12216 let rejection = Frame::build(
12217 FrameType::Error,
12218 control_flags(),
12219 0,
12220 0,
12221 bind.header.corr,
12222 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
12223 )
12224 .unwrap();
12225 handler
12226 .handle_control_frame(&module_ctx, rejection)
12227 .await
12228 .unwrap();
12229
12230 let response = route_task.await.unwrap();
12231 assert_eq!(parse_error(&response[0])["code"], hostile_code);
12232 let counters = handler.counters().snapshot();
12233 assert_eq!(
12234 counters["route_open_refused_by_code"],
12235 json!({ "module_rejected": 1 })
12236 );
12237 assert!(counters["route_open_refused_by_code"]
12238 .get(hostile_code)
12239 .is_none());
12240
12241 let event = capture
12242 .events()
12243 .into_iter()
12244 .find(|event| {
12245 event.target == "control"
12246 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
12247 })
12248 .expect("route.open module-rejection refusal event");
12249 let logged = event.fields.get("module_code").expect("module_code field");
12250 assert!(!logged.bytes().any(|byte| byte < 0x20));
12251 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
12252 }
12253
12254 #[tokio::test]
12255 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
12256 let registry = Arc::new(Registry::default());
12257 let supervisor_handle = SupervisorHandle::new();
12258 let missing_program = std::env::temp_dir().join(format!(
12259 "subc-route-open-missing-program-{}",
12260 std::process::id()
12261 ));
12262 let supervisor =
12263 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
12264 .with_handle(supervisor_handle.clone());
12265 let module = supervisor
12266 .supervise_configured(
12267 ModuleSpec {
12268 module_id: "failed".to_string(),
12269 program: missing_program,
12270 args: Vec::new(),
12271 env: Vec::new(),
12272 reserved: false,
12273 reserved_prefixes: Vec::new(),
12274 protocol: ModuleProtocol::Subc,
12275 overlap: Default::default(),
12276 },
12277 true,
12278 )
12279 .unwrap();
12280 assert_eq!(module.state().unwrap(), ModuleState::Failed);
12281
12282 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
12283 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
12284 let response = handler
12285 .handle_control_frame(
12286 &ctx,
12287 route_open_frame(305, "failed", unique_project_root("failed")),
12288 )
12289 .await
12290 .unwrap();
12291
12292 assert_eq!(response[0].header.ty, FrameType::Error);
12293 let error = parse_error(&response[0]);
12294 assert_eq!(error["code"], "target_unavailable");
12295 assert!(error["message"]
12296 .as_str()
12297 .unwrap()
12298 .contains("state=failed, enabled=true, live=false"));
12299 }
12300
12301 #[tokio::test]
12302 async fn route_open_role_mismatch_remains_target_unavailable() {
12303 let registry = Arc::new(Registry::default());
12304 let handler = ControlHandler::new(Arc::clone(®istry));
12305 handler
12306 .handle_control(
12307 ConnectionId::new(41),
12308 non_routable_hello_frame_with_control_ops("health-only", 306, None),
12309 )
12310 .unwrap();
12311
12312 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
12313 let response = handler
12314 .handle_control_frame(
12315 &ctx,
12316 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
12317 )
12318 .await
12319 .unwrap();
12320
12321 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12322 assert!(parse_error(&response[0])["message"]
12323 .as_str()
12324 .unwrap()
12325 .contains("does not provide the requested target"));
12326 }
12327
12328 #[tokio::test]
12329 async fn route_open_inactive_registration_remains_target_unavailable() {
12330 let registry = Arc::new(Registry::default());
12331 let handler = ControlHandler::new(Arc::clone(®istry));
12332 handler
12333 .handle_control(
12334 ConnectionId::new(43),
12335 hello_frame("inactive", PROTOCOL_VERSION, 308),
12336 )
12337 .unwrap();
12338 assert!(registry
12339 .set_module_state_for_test("inactive", ChannelState::Closed)
12340 .unwrap());
12341
12342 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
12343 let response = handler
12344 .handle_control_frame(
12345 &ctx,
12346 route_open_frame(309, "inactive", unique_project_root("inactive")),
12347 )
12348 .await
12349 .unwrap();
12350
12351 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12352 assert!(parse_error(&response[0])["message"]
12353 .as_str()
12354 .unwrap()
12355 .contains("is not active"));
12356 }
12357
12358 #[tokio::test]
12359 async fn late_health_reply_is_recorded_through_the_module_response_path() {
12360 let registry = Arc::new(Registry::default());
12361 let forwarding = Arc::new(ForwardingTable::default());
12362 let supervisor_handle = SupervisorHandle::new();
12363 let supervisor =
12364 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
12365 .with_forwarding(Arc::clone(&forwarding))
12366 .with_handle(supervisor_handle.clone());
12367 let module = supervisor
12368 .supervise_configured(
12369 crate::ModuleSpec {
12370 module_id: "late-health-response".to_string(),
12371 program: PathBuf::from("disabled-module"),
12372 args: Vec::new(),
12373 env: Vec::new(),
12374 reserved: false,
12375 reserved_prefixes: Vec::new(),
12376 protocol: ModuleProtocol::Subc,
12377 overlap: Default::default(),
12378 },
12379 false,
12380 )
12381 .unwrap();
12382 let handler =
12383 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12384 .with_supervisor(supervisor_handle);
12385 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
12386 handler
12387 .handle_control_frame(
12388 &module_ctx,
12389 hello_frame_with_control_ops(
12390 "late-health-response",
12391 PROTOCOL_VERSION,
12392 7,
12393 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12394 ),
12395 )
12396 .await
12397 .unwrap();
12398 let probe_started_at = Instant::now() - Duration::from_millis(80);
12399 let pending = forwarding
12400 .begin_health_probe_rpc_for(
12401 "late-health-response",
12402 MODULE_CONTROL_OP_HEALTH_CHECK,
12403 probe_started_at,
12404 Instant::now() - Duration::from_millis(1),
12405 )
12406 .unwrap();
12407 assert!(forwarding
12408 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
12409 .unwrap());
12410
12411 let responses = handler
12412 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
12413 .await
12414 .unwrap();
12415
12416 assert!(responses.is_empty());
12417 let health = module.status().unwrap().health;
12418 assert_eq!(health.late_answer_count, 1);
12419 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
12420 }
12421
12422 #[tokio::test]
12423 async fn health_probe_timeout_and_module_death_are_typed() {
12424 let registry = Arc::new(Registry::default());
12425 let forwarding = Arc::new(ForwardingTable::default());
12426 let handler =
12427 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12428 .with_health_probe_timeout(Duration::from_millis(50));
12429 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12430 hello_via_sink(
12431 &handler,
12432 &module_ctx,
12433 &mut module_rx,
12434 hello_frame_with_control_ops(
12435 "aft",
12436 PROTOCOL_VERSION,
12437 7,
12438 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
12439 ),
12440 )
12441 .await;
12442
12443 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12444 let responses = handler
12445 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
12446 .await
12447 .unwrap();
12448 assert_eq!(responses[0].header.ty, FrameType::Error);
12449 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
12450 let _ = module_rx.try_recv();
12451
12452 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
12453 let health_handler = handler.clone();
12454 let death_task = tokio::spawn(async move {
12455 health_handler
12456 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
12457 .await
12458 .unwrap()
12459 });
12460 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12461 .await
12462 .unwrap()
12463 .unwrap();
12464 handler
12465 .cleanup_connection(module_ctx.connection_id)
12466 .unwrap();
12467 let responses = death_task.await.unwrap();
12468 assert_eq!(responses[0].header.ty, FrameType::Error);
12469 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
12470 }
12471
12472 #[test]
12473 fn hello_requires_exact_protocol_version() {
12474 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
12475 let registry = Arc::new(Registry::default());
12476 let handler = ControlHandler::new(Arc::clone(®istry));
12477 let responses = handler
12478 .handle_control(
12479 ConnectionId::new(connection),
12480 hello_frame("aft", offered, 9),
12481 )
12482 .unwrap();
12483
12484 assert_eq!(responses.len(), 1);
12485 assert_eq!(responses[0].header.ty, FrameType::Error);
12486 let error = parse_error(&responses[0]);
12487 assert_eq!(error["code"], "version_unsupported");
12488 assert!(registry.get_module("aft").unwrap().is_none());
12489 assert_eq!(registry.active_registration_count().unwrap(), 0);
12490 }
12491 }
12492
12493 #[test]
12494 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
12495 let registry = Arc::new(Registry::default());
12496 let forwarding = Arc::new(ForwardingTable::default());
12497 let handler =
12498 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12499 let module_connection = ConnectionId::new(301);
12500 let registration = registry
12501 .register_with_control_ops(
12502 manifest("aft-push", PROTOCOL_VERSION),
12503 PROTOCOL_VERSION,
12504 module_connection,
12505 module_baseline_control_ops(),
12506 )
12507 .unwrap();
12508 let (module_tx, _module_rx) = mpsc::channel(8);
12509 let endpoint = forwarding
12510 .register_module_connection(
12511 module_connection,
12512 "aft-push".to_string(),
12513 PROTOCOL_VERSION,
12514 manifest_concurrency(®istration.manifest),
12515 FrameSink::new(module_tx),
12516 )
12517 .unwrap();
12518
12519 let unknown = Frame::build(
12521 FrameType::Push,
12522 control_flags(),
12523 0,
12524 0,
12525 5,
12526 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
12527 )
12528 .unwrap();
12529 let out = handler.handle_status_update(endpoint, unknown).unwrap();
12530 assert!(
12531 out.is_empty(),
12532 "unknown push op must be ignored, got {out:?}"
12533 );
12534
12535 let malformed = Frame::build(
12537 FrameType::Push,
12538 control_flags(),
12539 0,
12540 0,
12541 6,
12542 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
12543 )
12544 .unwrap();
12545 let out = handler.handle_status_update(endpoint, malformed).unwrap();
12546 assert_eq!(out.len(), 1);
12547 assert_eq!(out[0].header.ty, FrameType::Error);
12548 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
12549 }
12550
12551 #[test]
12552 fn hello_rejected_when_connection_already_owns_client_routes() {
12553 let registry = Arc::new(Registry::default());
12554 let forwarding = Arc::new(ForwardingTable::default());
12555 let handler =
12556 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12557 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
12559 let client_connection = ConnectionId::new(202);
12560
12561 let responses = handler
12564 .handle_control(
12565 client_connection,
12566 hello_frame("aft-second", PROTOCOL_VERSION, 9),
12567 )
12568 .unwrap();
12569 assert_eq!(responses[0].header.ty, FrameType::Error);
12570 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
12571 assert!(registry.get_module("aft-second").unwrap().is_none());
12572 }
12573
12574 #[tokio::test]
12575 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
12576 let registry = Arc::new(Registry::default());
12577 let forwarding = Arc::new(ForwardingTable::default());
12578 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
12579 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
12580 hello_via_sink(
12581 &handler,
12582 &module_ctx,
12583 &mut module_rx,
12584 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
12585 )
12586 .await;
12587 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
12588 let pending = forwarding
12589 .begin_route_bind_relay_for_test(
12590 client_ctx.connection_id,
12591 client_ctx.egress.clone(),
12592 2,
12593 "alpha",
12594 )
12595 .unwrap();
12596 forwarding
12597 .complete_pending_relay(
12598 module_ctx.connection_id,
12599 pending.corr,
12600 RouteBindRelayOutcome::Accepted,
12601 )
12602 .unwrap();
12603 client_rx.try_recv().unwrap();
12604 for module_id in ["beta", "alpha"] {
12605 let replies = handler
12606 .handle_control_frame(
12607 &module_ctx,
12608 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
12609 )
12610 .await
12611 .unwrap();
12612 assert_eq!(replies.len(), 1, "second HELLO must be refused");
12613 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
12614 }
12615 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
12616 assert!(registry.get_module("beta").unwrap().is_none());
12617 assert!(matches!(
12618 forwarding
12619 .lookup_data_route(
12620 client_ctx.connection_id,
12621 pending.client_channel,
12622 pending.client_epoch,
12623 )
12624 .unwrap(),
12625 DataRoute::Client(DataRouteState::Bound(_))
12626 ));
12627 assert!(handler
12628 .hello_launch_nonces
12629 .lock()
12630 .unwrap()
12631 .presented(module_ctx.connection_id, Some("alpha-nonce")));
12632 assert!(module_rx.try_recv().is_err());
12633 }
12634
12635 #[test]
12636 fn reserved_module_hello_requires_matching_launch_nonce() {
12637 let registry = Arc::new(Registry::default());
12638 let supervisor = SupervisorHandle::new();
12639 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
12642 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12643
12644 let no_nonce = handler
12646 .handle_control(
12647 ConnectionId::new(1),
12648 hello_frame("vault", PROTOCOL_VERSION, 1),
12649 )
12650 .unwrap();
12651 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
12652 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
12653 assert!(registry.get_module("vault").unwrap().is_none());
12654
12655 let wrong = handler
12657 .handle_control(
12658 ConnectionId::new(2),
12659 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
12660 )
12661 .unwrap();
12662 assert_eq!(wrong[0].header.ty, FrameType::Error);
12663 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
12664 assert!(registry.get_module("vault").unwrap().is_none());
12665
12666 let ok = handler
12668 .handle_control(
12669 ConnectionId::new(3),
12670 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
12671 )
12672 .unwrap();
12673 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
12674 assert!(registry.get_module("vault").unwrap().is_some());
12675 }
12676
12677 #[test]
12678 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
12679 let registry = Arc::new(Registry::default());
12680 let supervisor = SupervisorHandle::new();
12681 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12682 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12683 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12684
12685 let squat = handler
12686 .handle_control(
12687 ConnectionId::new(1),
12688 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
12689 )
12690 .unwrap();
12691 assert_eq!(squat[0].header.ty, FrameType::Error);
12692 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
12693 assert!(parse_error(&squat[0])["message"]
12694 .as_str()
12695 .unwrap()
12696 .contains("fed:"));
12697
12698 let accepted_peer = handler
12699 .handle_control(
12700 ConnectionId::new(2),
12701 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
12702 )
12703 .unwrap();
12704 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
12705
12706 let accepted_short = handler
12707 .handle_control(
12708 ConnectionId::new(3),
12709 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
12710 )
12711 .unwrap();
12712 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
12713
12714 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
12715 let response = handler
12716 .handle_control(
12717 ConnectionId::new(conn),
12718 hello_frame(module_id, PROTOCOL_VERSION, conn),
12719 )
12720 .unwrap();
12721 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
12722 }
12723 }
12724
12725 #[test]
12726 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
12727 let registry = Arc::new(Registry::default());
12728 let supervisor = SupervisorHandle::new();
12729 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
12730 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
12731 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
12732 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
12733
12734 let owner_nonce = handler
12735 .handle_control(
12736 ConnectionId::new(1),
12737 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
12738 )
12739 .unwrap();
12740 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
12741 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
12742 assert!(registry.get_module("fed:special").unwrap().is_none());
12743
12744 let exact_nonce = handler
12745 .handle_control(
12746 ConnectionId::new(2),
12747 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
12748 )
12749 .unwrap();
12750 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
12751 assert!(registry.get_module("fed:special").unwrap().is_some());
12752 }
12753
12754 #[test]
12755 fn non_reserved_module_ignores_launch_nonce() {
12756 let registry = Arc::new(Registry::default());
12757 let handler = ControlHandler::new(Arc::clone(®istry));
12760 let no_nonce = handler
12761 .handle_control(
12762 ConnectionId::new(1),
12763 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
12764 )
12765 .unwrap();
12766 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
12767 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
12768
12769 let echoed_nonce = handler
12770 .handle_control(
12771 ConnectionId::new(2),
12772 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
12773 )
12774 .unwrap();
12775 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
12776 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
12777 }
12778
12779 #[test]
12780 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
12781 let handler = ControlHandler::default();
12782 let conn = ConnectionId::new(1);
12783 let malformed = Frame::build(
12784 FrameType::Hello,
12785 control_flags(),
12786 0,
12787 0,
12788 3,
12789 b"{not json".to_vec(),
12790 )
12791 .unwrap();
12792
12793 let error = handler.handle_control(conn, malformed).unwrap();
12794 assert_eq!(error[0].header.ty, FrameType::Error);
12795 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
12796
12797 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
12798 let pong = handler.handle_control(conn, ping).unwrap();
12799 assert_eq!(pong[0].header.ty, FrameType::Pong);
12800 assert_eq!(pong[0].header.corr, 4);
12801 }
12802
12803 #[test]
12804 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
12805 let registry = Arc::new(Registry::default());
12806 let handler = ControlHandler::new(Arc::clone(®istry));
12807
12808 handler
12809 .handle_control(
12810 ConnectionId::new(1),
12811 hello_frame("aft", PROTOCOL_VERSION, 1),
12812 )
12813 .unwrap();
12814 let duplicate = handler
12815 .handle_control(
12816 ConnectionId::new(2),
12817 hello_frame("aft", PROTOCOL_VERSION, 2),
12818 )
12819 .unwrap();
12820
12821 assert_eq!(duplicate[0].header.ty, FrameType::Error);
12822 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
12823 let registration = registry.get_module("aft").unwrap().unwrap();
12824 assert_eq!(registration.connection_id, ConnectionId::new(1));
12825 }
12826
12827 #[test]
12828 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
12829 let registry = Arc::new(Registry::default());
12830 let forwarding = Arc::new(ForwardingTable::default());
12831 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
12832 let handler =
12833 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12834 .with_process_liveness(process_liveness);
12835 let (ctx, route_channel, route_epoch) =
12836 bind_liveness_route(®istry, &forwarding, "aft-dead");
12837 let responses = handler
12838 .handle_route_poll(
12839 &ctx,
12840 route_poll_frame(41, PollKind::Liveness, route_channel),
12841 route_channel,
12842 route_epoch,
12843 PollKind::Liveness,
12844 )
12845 .unwrap();
12846
12847 assert_eq!(responses.len(), 1);
12848 assert_eq!(responses[0].header.ty, FrameType::Response);
12849 assert_route_poll_liveness(&responses[0], false);
12850 }
12851
12852 #[test]
12853 fn liveness_poll_without_process_source_uses_bound_route() {
12854 let registry = Arc::new(Registry::default());
12855 let forwarding = Arc::new(ForwardingTable::default());
12856 let handler =
12857 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12858 let (ctx, route_channel, route_epoch) =
12859 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
12860 let responses = handler
12861 .handle_route_poll(
12862 &ctx,
12863 route_poll_frame(42, PollKind::Liveness, route_channel),
12864 route_channel,
12865 route_epoch,
12866 PollKind::Liveness,
12867 )
12868 .unwrap();
12869
12870 assert_route_poll_liveness(&responses[0], true);
12871 }
12872
12873 #[test]
12874 fn liveness_poll_untracked_process_source_uses_bound_route() {
12875 let registry = Arc::new(Registry::default());
12876 let forwarding = Arc::new(ForwardingTable::default());
12877 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
12878 let handler =
12879 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12880 .with_process_liveness(process_liveness);
12881 let (ctx, route_channel, route_epoch) =
12882 bind_liveness_route(®istry, &forwarding, "aft-untracked");
12883 let responses = handler
12884 .handle_route_poll(
12885 &ctx,
12886 route_poll_frame(43, PollKind::Liveness, route_channel),
12887 route_channel,
12888 route_epoch,
12889 PollKind::Liveness,
12890 )
12891 .unwrap();
12892
12893 assert_route_poll_liveness(&responses[0], true);
12894 }
12895
12896 #[tokio::test]
12897 async fn unknown_op_returns_unknown_control_op() {
12898 let handler = ControlHandler::default();
12899 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
12900 let request = Frame::build(
12901 FrameType::Request,
12902 control_flags(),
12903 0,
12904 0,
12905 55,
12906 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
12907 )
12908 .unwrap();
12909
12910 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12911
12912 assert_eq!(response.len(), 1);
12913 assert_eq!(response[0].header.ty, FrameType::Error);
12914 assert_eq!(response[0].header.corr, 55);
12915 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
12916 }
12917
12918 #[tokio::test]
12919 async fn supervisor_provenance_rejects_unknown_exact_module() {
12920 let handler = ControlHandler::default();
12921 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
12922 let request = Frame::build(
12923 FrameType::Request,
12924 control_flags(),
12925 0,
12926 0,
12927 57,
12928 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
12929 )
12930 .unwrap();
12931
12932 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
12933
12934 assert_eq!(response.len(), 1);
12935 assert_eq!(response[0].header.ty, FrameType::Error);
12936 assert_eq!(response[0].header.corr, 57);
12937 let error = parse_error(&response[0]);
12938 assert_eq!(error["code"], "unknown_module");
12939 assert_eq!(error["message"], "module_id 'missing' is not supervised");
12940 }
12941
12942 #[test]
12943 fn provenance_probe_override_keeps_handler_tests_deterministic() {
12944 let expected = subc_control::RunningImageAgreement::Unavailable {
12945 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12946 };
12947 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
12948 assert_eq!(handler.provenance_probe_override, Some(expected));
12949 }
12950
12951 #[test]
12952 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
12953 let verdict = reload_verdict(
12954 std::path::Path::new("/bin/new"),
12955 Some(std::path::Path::new("/bin/old")),
12956 subc_control::RunningImageAgreement::Unavailable {
12957 reason: subc_control::RunningImageUnavailableReason::HashFailed,
12958 },
12959 );
12960 assert!(matches!(
12961 verdict.path,
12962 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
12963 if configured == std::path::Path::new("/bin/new")
12964 && spawned_from == std::path::Path::new("/bin/old")
12965 ));
12966 }
12967
12968 #[test]
12969 fn launch_spec_without_running_digest_is_unavailable_not_match() {
12970 use subc_control::{
12971 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
12972 };
12973 let configured = crate::supervise::ModuleSpec {
12974 module_id: "legacy".into(),
12975 program: "/module".into(),
12976 args: Vec::new(),
12977 env: Vec::new(),
12978 reserved: false,
12979 reserved_prefixes: Vec::new(),
12980 protocol: subc_control::ModuleProtocol::Subc,
12981 overlap: Default::default(),
12982 };
12983 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
12984 assert_eq!(
12985 launch_spec_agreement(&configured, false, Some(&digest)),
12986 Agreement::Unavailable {
12987 reason: Reason::NotRunning
12988 }
12989 );
12990 assert_eq!(
12991 launch_spec_agreement(&configured, true, None),
12992 Agreement::Unavailable {
12993 reason: Reason::SpawnedSpecUnavailable
12994 }
12995 );
12996 assert_eq!(
12997 launch_spec_agreement(&configured, true, Some(&digest)),
12998 Agreement::Match
12999 );
13000 }
13001
13002 #[cfg(windows)]
13003 #[test]
13004 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
13005 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
13006 let configured = dir.join("module.exe");
13007 std::fs::write(&configured, b"executable path fixture").unwrap();
13008 let spawned = std::fs::canonicalize(&configured).unwrap();
13009 assert_ne!(
13010 configured, spawned,
13011 "fixture must exercise different path spellings"
13012 );
13013 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
13014 let lower_case = configured.to_string_lossy().to_lowercase();
13015 for spelling in [
13016 &configured,
13017 Path::new(&forward_slashes),
13018 Path::new(&lower_case),
13019 ] {
13020 let verdict = reload_verdict(
13021 spelling,
13022 Some(&spawned),
13023 subc_control::RunningImageAgreement::Unavailable {
13024 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13025 },
13026 );
13027 assert_eq!(
13028 verdict.path,
13029 ReloadPathAgreement::Match,
13030 "configured {spelling:?}, spawned {spawned:?}"
13031 );
13032 }
13033 }
13034
13035 #[cfg(windows)]
13036 #[test]
13037 fn windows_reload_path_keeps_different_existing_program_pending() {
13038 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
13039 let old = dir.join("old.exe");
13040 let new = dir.join("new.exe");
13041 std::fs::write(&old, b"same executable content").unwrap();
13042 std::fs::copy(&old, &new).unwrap();
13043 let spawned = std::fs::canonicalize(&old).unwrap();
13044 let image = subc_control::RunningImageAgreement::Unavailable {
13045 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13046 };
13047 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
13048 assert!(
13049 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
13050 if configured == new && spawned_from == spawned)
13051 );
13052 let missing = dir.join("missing.exe");
13053 assert!(matches!(
13054 reload_verdict(&missing, Some(&spawned), image).path,
13055 ReloadPathAgreement::Mismatch { .. }
13056 ));
13057 }
13058
13059 #[test]
13060 fn reload_verdict_detects_replaced_image_at_same_path() {
13061 let image = subc_control::RunningImageAgreement::Mismatch {
13062 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
13063 digest: "old".into(),
13064 },
13065 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
13066 digest: "new".into(),
13067 },
13068 };
13069 let verdict = reload_verdict(
13070 std::path::Path::new("/bin/same"),
13071 Some(std::path::Path::new("/bin/same")),
13072 image.clone(),
13073 );
13074 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13075 assert_eq!(verdict.image, image);
13076 }
13077
13078 #[test]
13079 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
13080 let image = subc_control::RunningImageAgreement::Unavailable {
13081 reason: subc_control::RunningImageUnavailableReason::NotRunning,
13082 };
13083 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
13084 assert_eq!(
13085 verdict.path,
13086 subc_control::ReloadPathAgreement::Unavailable {
13087 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
13088 }
13089 );
13090 assert_eq!(verdict.image, image);
13091
13092 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
13093 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
13094 };
13095 let verdict = reload_verdict(
13096 std::path::Path::new("/bin/same"),
13097 Some(std::path::Path::new("/bin/same")),
13098 unconfirmed.clone(),
13099 );
13100 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13101 assert_eq!(verdict.image, unconfirmed);
13102 }
13103
13104 #[test]
13105 fn reload_verdict_preserves_each_image_unavailability_reason() {
13106 use subc_control::RunningImageUnavailableReason as Reason;
13107
13108 for reason in [
13109 Reason::NotRunning,
13110 Reason::UnsupportedPlatform,
13111 Reason::RunningExecutableUnreadable,
13112 Reason::SpawnedPathUnreadable,
13113 Reason::HashFailed,
13114 Reason::ProcessIdentityUnconfirmed,
13115 Reason::Unknown("future_probe_reason".to_string()),
13116 ] {
13117 let image = subc_control::RunningImageAgreement::Unavailable {
13118 reason: reason.clone(),
13119 };
13120 let verdict = reload_verdict(
13121 std::path::Path::new("/bin/same"),
13122 Some(std::path::Path::new("/bin/same")),
13123 image.clone(),
13124 );
13125 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
13126 assert_eq!(verdict.image, image, "{reason:?}");
13127 }
13128 }
13129
13130 #[tokio::test]
13131 async fn malformed_control_bodies_return_invalid_control_body() {
13132 let handler = ControlHandler::default();
13133 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
13134
13135 for (corr, body) in [
13136 (56, br#"{"route_channel":1}"#.as_slice()),
13137 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
13138 (
13139 58,
13140 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
13141 ),
13142 ] {
13143 let request = Frame::build(
13144 FrameType::Request,
13145 control_flags(),
13146 0,
13147 0,
13148 corr,
13149 body.to_vec(),
13150 )
13151 .unwrap();
13152 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
13153
13154 assert_eq!(response.len(), 1);
13155 assert_eq!(response[0].header.ty, FrameType::Error);
13156 assert_eq!(response[0].header.corr, corr);
13157 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
13158 }
13159 }
13160
13161 #[tokio::test]
13162 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
13163 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13164 let registry = Arc::new(Registry::default());
13165 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13166 let router = Router::with_control_handler(Arc::clone(&control));
13167 let connection = router.begin_connection();
13168 let (ctx, mut rx) = route_ctx(connection.id());
13169
13170 router
13171 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
13172 .await
13173 .unwrap();
13174 let response = rx.recv().await.unwrap();
13175 let ack = parse_ack(&response);
13176 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13177 let channel = 1;
13178
13179 let goodbye =
13180 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
13181 router.route_for_connection(&ctx, goodbye).await.unwrap();
13182 assert!(rx.try_recv().is_err());
13183 assert!(registry.get_module("aft").unwrap().is_none());
13184
13185 router
13186 .route_for_connection(&ctx, channel_request(channel, 13))
13187 .await
13188 .unwrap();
13189 let error_frame = rx.recv().await.unwrap();
13190 assert_eq!(error_frame.header.ty, FrameType::Error);
13191 assert_eq!(error_frame.header.channel, channel);
13192 let captured = crate::router::test_log::captured_logs(&logs);
13193 assert_eq!(
13194 captured
13195 .lines()
13196 .filter(|line| {
13197 line.contains("module_id=aft")
13198 && line.contains("reason=explicit_goodbye")
13199 && line.contains("module registration ended")
13200 })
13201 .count(),
13202 1,
13203 "unexpected GOODBYE registry log: {captured}"
13204 );
13205 }
13206
13207 #[tokio::test]
13208 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
13209 let registry = Arc::new(Registry::default());
13210 let handler = ControlHandler::new(registry).with_capability_config(
13211 [("prov".to_string(), true), ("cons".to_string(), true)],
13212 BTreeMap::new(),
13213 );
13214 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
13215 register_capability_manifest(
13216 &handler,
13217 &provider_ctx,
13218 &mut provider_rx,
13219 capability_manifest("prov", &["thing/v1"], &[]),
13220 1,
13221 )
13222 .await;
13223 let mut consumer = capability_manifest("cons", &[], &[]);
13224 consumer.capabilities.as_mut().unwrap().requires.push(
13225 subc_protocol::manifest::CapabilityRequirement {
13226 capability: "thing/v1".to_string(),
13227 need: subc_protocol::manifest::CapabilityNeed::Required,
13228 },
13229 );
13230 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
13231 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
13232 assert_eq!(
13233 handler.capability_evaluator.verdict("cons", "thing/v1"),
13234 Some(CapabilityVerdict::Provided)
13235 );
13236 let (mut client_rx, _) = open_route_for_capability_test(
13237 &handler,
13238 &provider_ctx,
13239 &mut provider_rx,
13240 703,
13241 3,
13242 "prov",
13243 None,
13244 )
13245 .await;
13246 let goodbye =
13247 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
13248 handler
13249 .handle_control_frame(&provider_ctx, goodbye)
13250 .await
13251 .unwrap();
13252 assert_eq!(
13253 handler.capability_evaluator.verdict("cons", "thing/v1"),
13254 Some(CapabilityVerdict::NeverProvided)
13255 );
13256 let closed = client_rx
13257 .try_recv()
13258 .expect("GOODBYE pushes route.closed before route GOODBYE");
13259 assert!(
13260 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
13261 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
13262 );
13263 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
13264 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
13265 }
13266
13267 struct RouteClosedFixture {
13268 handler: ControlHandler,
13269 #[cfg(unix)]
13270 supervisor: Supervisor,
13271 module_ctx: RouteCtx,
13272 #[cfg(unix)]
13273 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13274 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
13275 route: Arc<crate::forwarding::RouteBinding>,
13276 }
13277
13278 async fn route_closed_fixture() -> RouteClosedFixture {
13279 let registry = Arc::new(Registry::default());
13280 let handler = ControlHandler::new(Arc::clone(®istry));
13281 #[cfg(unix)]
13282 let supervisor = Supervisor::new(registry, RestartPolicy::default())
13283 .with_forwarding(Arc::clone(&handler.forwarding));
13284 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
13285 register_capability_manifest(
13286 &handler,
13287 &module_ctx,
13288 &mut module_rx,
13289 capability_manifest("prov", &[], &[]),
13290 1,
13291 )
13292 .await;
13293 let (client_rx, bind) = open_route_for_capability_test(
13294 &handler,
13295 &module_ctx,
13296 &mut module_rx,
13297 702,
13298 2,
13299 "prov",
13300 None,
13301 )
13302 .await;
13303 let ModuleControlRequest::RouteBind {
13304 route_channel: channel,
13305 epoch: route_epoch,
13306 ..
13307 } = bind
13308 else {
13309 panic!("expected route.bind");
13310 };
13311 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
13312 handler
13313 .forwarding
13314 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
13315 .unwrap()
13316 else {
13317 panic!("the acknowledged route must be bound");
13318 };
13319 RouteClosedFixture {
13320 handler,
13321 #[cfg(unix)]
13322 supervisor,
13323 module_ctx,
13324 #[cfg(unix)]
13325 module_rx,
13326 client_rx,
13327 route,
13328 }
13329 }
13330
13331 impl RouteClosedFixture {
13332 #[cfg(unix)]
13333 async fn cut(&mut self) {
13334 let notices = async {
13335 let mut draining = self.module_rx.recv().await.unwrap();
13336 assert!(matches!(
13337 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
13338 &draining.body
13339 )
13340 .unwrap(),
13341 subc_protocol::session::ModuleControlCommand::Draining {
13342 reason: RouteCloseReason::Restart,
13343 ..
13344 }
13345 ));
13346 draining.flushed.take().unwrap().send(()).unwrap();
13347 let mut closing = self.client_rx.recv().await.unwrap();
13348 assert!(matches!(
13349 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
13350 ClientControlPush::RouteClosing {
13351 reason: RouteCloseReason::Restart,
13352 ..
13353 }
13354 ));
13355 closing.flushed.take().unwrap().send(()).unwrap();
13356 };
13357 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
13358 result.unwrap();
13359 }
13360
13361 fn disconnect(&mut self) -> ClientControlPush {
13362 self.handler
13363 .cleanup_connection(self.module_ctx.connection_id)
13364 .unwrap();
13365 let closed = self.client_rx.try_recv().unwrap();
13366 assert_eq!(closed.header.ty, FrameType::Push);
13367 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
13368 assert!(matches!(
13369 &push,
13370 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
13371 if channels == &[self.route.client_channel]
13372 ));
13373 assert_eq!(
13374 self.client_rx.try_recv().unwrap().header.ty,
13375 FrameType::Goodbye
13376 );
13377 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
13378 push
13379 }
13380 }
13381
13382 #[cfg(unix)]
13383 #[tokio::test(start_paused = true)]
13384 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
13385 let mut fixture = route_closed_fixture().await;
13386 fixture.cut().await;
13387 assert!(matches!(
13388 fixture.disconnect(),
13389 ClientControlPush::RouteClosed {
13390 reason: RouteCloseReason::Restart,
13391 drained: true,
13392 ..
13393 }
13394 ));
13395 }
13396
13397 #[cfg(unix)]
13398 #[tokio::test(start_paused = true)]
13399 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
13400 let mut fixture = route_closed_fixture().await;
13401 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
13402 let started = tokio::time::Instant::now();
13403 fixture.cut().await;
13404 assert!(started.elapsed() >= Duration::from_secs(2));
13405 assert_eq!(fixture.route.flow.in_flight(), 1);
13406 assert!(matches!(
13407 fixture.disconnect(),
13408 ClientControlPush::RouteClosed {
13409 reason: RouteCloseReason::Restart,
13410 drained: false,
13411 ..
13412 }
13413 ));
13414 }
13415
13416 #[tokio::test]
13417 async fn module_connection_crash_reports_crash_and_not_drained() {
13418 let mut fixture = route_closed_fixture().await;
13419 assert!(matches!(
13423 fixture.disconnect(),
13424 ClientControlPush::RouteClosed {
13425 reason: RouteCloseReason::Crash,
13426 drained: false,
13427 ..
13428 }
13429 ));
13430 }
13431
13432 #[tokio::test]
13433 async fn dropping_router_connection_releases_registration() {
13434 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13435 let registry = Arc::new(Registry::default());
13436 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
13437 let router = Router::with_control_handler(Arc::clone(&control));
13438 let connection = router.begin_connection();
13439 let connection_id = connection.id();
13440 let (ctx, mut rx) = route_ctx(connection_id);
13441
13442 router
13443 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
13444 .await
13445 .unwrap();
13446 let response = rx.recv().await.unwrap();
13447 let ack = parse_ack(&response);
13448 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
13449 assert!(registry.get_module("aft").unwrap().is_some());
13450
13451 drop(connection);
13452
13453 assert!(registry.get_module("aft").unwrap().is_none());
13454 assert_eq!(registry.active_registration_count().unwrap(), 0);
13455
13456 control
13457 .cleanup_connection(ConnectionId::new(u64::MAX))
13458 .unwrap();
13459 let captured = crate::router::test_log::captured_logs(&logs);
13460 println!("captured registry lifecycle logs:\n{captured}");
13461 let events: Vec<_> = captured
13462 .lines()
13463 .filter(|line| {
13464 line.contains("module registered module_id=aft ")
13465 || line.contains("module registration ended")
13466 })
13467 .collect();
13468 assert_eq!(
13469 events.len(),
13470 2,
13471 "unexpected registry lifecycle logs: {captured}"
13472 );
13473 assert!(events[0].contains("module registered module_id=aft "));
13474 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
13475 assert!(events[1].contains(&format!(
13476 "module_id=aft connection_id={}",
13477 connection_id.get()
13478 )));
13479 assert!(events[1].contains("reason=connection_closed"));
13480 assert!(events[1].contains("module registration ended"));
13481 assert_eq!(
13482 captured
13483 .lines()
13484 .filter(|line| line.contains("module registered module_id=aft "))
13485 .count(),
13486 1,
13487 "legacy registration admission line must appear once: {captured}"
13488 );
13489 }
13490
13491 #[tokio::test]
13492 async fn hello_registration_keeps_legacy_module_registered_line_once() {
13493 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
13494 let registry = Arc::new(Registry::default());
13495 let control = ControlHandler::new(Arc::clone(®istry));
13496 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
13497 hello_via_sink(
13498 &control,
13499 &ctx,
13500 &mut rx,
13501 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
13502 )
13503 .await;
13504
13505 let captured = crate::router::test_log::captured_logs(&logs);
13506 let admissions: Vec<_> = captured
13507 .lines()
13508 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
13509 .collect();
13510 assert_eq!(
13511 admissions.len(),
13512 1,
13513 "legacy admission line must remain exactly once: {captured}"
13514 );
13515 assert!(admissions[0].contains("routable_provider=true"));
13516 assert!(admissions[0].contains("connection_id=777"));
13517 }
13518
13519 fn capability_manifest(
13520 module_id: &str,
13521 provides: &[&str],
13522 must_never_reach: &[&str],
13523 ) -> ModuleManifest {
13524 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
13525 manifest.capabilities = Some(CapabilityDeclarations {
13526 provides: provides
13527 .iter()
13528 .map(|capability| (*capability).to_string())
13529 .collect(),
13530 requires: Vec::new(),
13531 must_never_reach: must_never_reach
13532 .iter()
13533 .map(|capability| (*capability).to_string())
13534 .collect(),
13535 });
13536 manifest
13537 }
13538
13539 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
13540 Frame::build(
13541 FrameType::Hello,
13542 control_flags(),
13543 0,
13544 0,
13545 corr,
13546 serde_json::to_vec(&ModuleHelloBody {
13547 protocol_ver: manifest.protocol_ver,
13548 manifest,
13549 control_ops: None,
13550 launch_nonce: None,
13551 })
13552 .expect("capability test HELLO serializes"),
13553 )
13554 .expect("capability test HELLO frame builds")
13555 }
13556
13557 fn catalog_update_with_capabilities_frame(
13558 corr: u64,
13559 capabilities: CapabilityDeclarations,
13560 ) -> Frame {
13561 Frame::build(
13562 FrameType::Request,
13563 control_flags(),
13564 0,
13565 0,
13566 corr,
13567 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
13568 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
13569 capabilities: Some(capabilities),
13570 ready: None,
13571 })
13572 .expect("capability catalog.update serializes"),
13573 )
13574 .expect("capability catalog.update frame builds")
13575 }
13576
13577 async fn register_capability_manifest(
13578 handler: &ControlHandler,
13579 ctx: &RouteCtx,
13580 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13581 manifest: ModuleManifest,
13582 corr: u64,
13583 ) {
13584 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
13585 }
13586
13587 async fn open_route_for_capability_test(
13588 handler: &ControlHandler,
13589 target_ctx: &RouteCtx,
13590 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
13591 client_connection_id: u64,
13592 corr: u64,
13593 target_module_id: &str,
13594 consumer_identity: Option<ConsumerIdentity>,
13595 ) -> (
13596 mpsc::Receiver<crate::router::OutboundFrame>,
13597 ModuleControlRequest,
13598 ) {
13599 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
13600 let route_handler = handler.clone();
13601 let target_module_id = target_module_id.to_string();
13602 let route_task = tokio::spawn(async move {
13603 route_handler
13604 .handle_control_frame(
13605 &client_ctx,
13606 route_open_frame_with_admission_facts(
13607 corr,
13608 &target_module_id,
13609 unique_project_root("admission-facts"),
13610 consumer_identity,
13611 None,
13612 ),
13613 )
13614 .await
13615 .expect("capability test route.open succeeds")
13616 });
13617 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
13618 .await
13619 .expect("capability test route.open must reach route.bind")
13620 .expect("target control receiver stays open");
13621 let bind_request: ModuleControlRequest =
13622 serde_json::from_slice(&bind.body).expect("route.bind decodes");
13623 handler
13624 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
13625 .await
13626 .expect("capability test route.bind ACK succeeds");
13627 assert!(route_task.await.expect("route.open task joins").is_empty());
13628 let opened = client_rx
13629 .recv()
13630 .await
13631 .expect("successful route.open publishes a response");
13632 assert!(matches!(
13633 serde_json::from_slice::<ClientControlResponse>(&opened.body),
13634 Ok(ClientControlResponse::RouteOpen { .. })
13635 ));
13636 (client_rx, bind_request)
13637 }
13638
13639 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) {
13640 assert_eq!(frame.header.ty, FrameType::Push);
13641 assert_eq!(frame.header.channel, 0);
13642 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
13643 .expect("route.closed control push decodes");
13644 let ClientControlPush::RouteClosed { channels, .. } = &push else {
13645 panic!("expected route.closed");
13646 };
13647 assert_eq!(channels.len(), 1, "exactly one violating route closed");
13648 let channels = channels.clone();
13649 assert_eq!(
13650 push,
13651 ClientControlPush::RouteClosed {
13652 module_id: target_module_id.to_string(),
13653 channels,
13654 reason: RouteCloseReason::CapabilityDenied,
13655 drained: false,
13656 abandoned: 0,
13657 excluded_subscriptions: 0,
13658 terminal: Some(false),
13659 }
13660 );
13661 }
13662
13663 #[tokio::test]
13664 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
13665 let registry = Arc::new(Registry::default());
13666 let forwarding = Arc::new(ForwardingTable::default());
13667 let supervisor = SupervisorHandle::new();
13668 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13669 let handler =
13670 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13671 .with_supervisor(supervisor);
13672 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
13673 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
13674 register_capability_manifest(
13675 &handler,
13676 &target_ctx,
13677 &mut target_rx,
13678 capability_manifest("target", &["credentials-provider/v1"], &[]),
13679 1,
13680 )
13681 .await;
13682 register_capability_manifest(
13683 &handler,
13684 &opener_ctx,
13685 &mut opener_rx,
13686 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13687 2,
13688 )
13689 .await;
13690
13691 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
13692 let replies = handler
13693 .handle_control_frame(
13694 &client_ctx,
13695 route_open_frame_with_admission_facts(
13696 3,
13697 "target",
13698 unique_project_root("admission-facts"),
13699 Some(ConsumerIdentity {
13700 module_id: "opener".to_string(),
13701 launch_nonce: "opener-nonce".to_string(),
13702 }),
13703 None,
13704 ),
13705 )
13706 .await
13707 .expect("denied route.open returns a typed frame");
13708 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
13709 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13710 assert!(
13711 target_rx.try_recv().is_err(),
13712 "forbidden route.open must not relay route.bind"
13713 );
13714 }
13715
13716 #[tokio::test]
13717 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
13718 let registry = Arc::new(Registry::default());
13719 let forwarding = Arc::new(ForwardingTable::default());
13720 let supervisor = SupervisorHandle::new();
13721 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13722 let handler =
13723 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13724 .with_supervisor(supervisor);
13725 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
13726 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
13727 register_capability_manifest(
13728 &handler,
13729 &target_ctx,
13730 &mut target_rx,
13731 capability_manifest("target", &["credentials-provider/v1"], &[]),
13732 1,
13733 )
13734 .await;
13735 register_capability_manifest(
13736 &handler,
13737 &old_opener_ctx,
13738 &mut old_opener_rx,
13739 capability_manifest("opener", &[], &[]),
13740 2,
13741 )
13742 .await;
13743 let (mut client_rx, _) = open_route_for_capability_test(
13744 &handler,
13745 &target_ctx,
13746 &mut target_rx,
13747 712,
13748 3,
13749 "target",
13750 Some(ConsumerIdentity {
13751 module_id: "opener".to_string(),
13752 launch_nonce: "opener-nonce".to_string(),
13753 }),
13754 )
13755 .await;
13756 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13757
13758 handler
13759 .cleanup_connection(old_opener_ctx.connection_id)
13760 .expect("old opener registration cleans up");
13761 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
13762 register_capability_manifest(
13763 &handler,
13764 &new_opener_ctx,
13765 &mut new_opener_rx,
13766 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13767 4,
13768 )
13769 .await;
13770
13771 assert_capability_denied_push(
13772 client_rx
13773 .try_recv()
13774 .expect("HELLO deny addition must emit route.closed")
13775 .frame,
13776 "target",
13777 );
13778 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13779 assert!(matches!(
13780 target_rx.try_recv(),
13781 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13782 ));
13783 }
13784
13785 #[tokio::test]
13786 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
13787 let registry = Arc::new(Registry::default());
13788 let forwarding = Arc::new(ForwardingTable::default());
13789 let supervisor = SupervisorHandle::new();
13790 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13791 let handler =
13792 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13793 .with_supervisor(supervisor);
13794 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
13795 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
13796 register_capability_manifest(
13797 &handler,
13798 &target_ctx,
13799 &mut target_rx,
13800 capability_manifest("target", &[], &[]),
13801 1,
13802 )
13803 .await;
13804 register_capability_manifest(
13805 &handler,
13806 &opener_ctx,
13807 &mut opener_rx,
13808 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13809 2,
13810 )
13811 .await;
13812 let (mut client_rx, _) = open_route_for_capability_test(
13813 &handler,
13814 &target_ctx,
13815 &mut target_rx,
13816 722,
13817 3,
13818 "target",
13819 Some(ConsumerIdentity {
13820 module_id: "opener".to_string(),
13821 launch_nonce: "opener-nonce".to_string(),
13822 }),
13823 )
13824 .await;
13825 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13826
13827 let replies = handler
13828 .handle_control_frame(
13829 &target_ctx,
13830 catalog_update_with_capabilities_frame(
13831 4,
13832 CapabilityDeclarations {
13833 provides: vec!["credentials-provider/v1".to_string()],
13834 requires: Vec::new(),
13835 must_never_reach: Vec::new(),
13836 },
13837 ),
13838 )
13839 .await
13840 .expect("claim catalog.update succeeds");
13841 assert!(matches!(
13842 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
13843 Ok(ModuleControlResponseToModule::CatalogUpdate {})
13844 ));
13845 assert_capability_denied_push(
13846 client_rx
13847 .try_recv()
13848 .expect("claim addition must emit route.closed")
13849 .frame,
13850 "target",
13851 );
13852 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
13853 assert!(matches!(
13854 target_rx.try_recv(),
13855 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
13856 ));
13857 }
13858
13859 #[tokio::test]
13860 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
13861 let registry = Arc::new(Registry::default());
13862 let forwarding = Arc::new(ForwardingTable::default());
13863 let supervisor = SupervisorHandle::new();
13864 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13865 let handler =
13866 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13867 .with_supervisor(supervisor);
13868 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
13869 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
13870 register_capability_manifest(
13871 &handler,
13872 &target_ctx,
13873 &mut target_rx,
13874 capability_manifest("target", &["credentials-provider/v1"], &[]),
13875 1,
13876 )
13877 .await;
13878 register_capability_manifest(
13879 &handler,
13880 &opener_ctx,
13881 &mut opener_rx,
13882 capability_manifest("opener", &[], &[]),
13883 2,
13884 )
13885 .await;
13886 let (mut client_rx, _) = open_route_for_capability_test(
13887 &handler,
13888 &target_ctx,
13889 &mut target_rx,
13890 732,
13891 3,
13892 "target",
13893 Some(ConsumerIdentity {
13894 module_id: "opener".to_string(),
13895 launch_nonce: "opener-nonce".to_string(),
13896 }),
13897 )
13898 .await;
13899 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13900
13901 handler
13902 .handle_control_frame(
13903 &target_ctx,
13904 catalog_update_with_capabilities_frame(
13905 4,
13906 CapabilityDeclarations {
13907 provides: Vec::new(),
13908 requires: Vec::new(),
13909 must_never_reach: Vec::new(),
13910 },
13911 ),
13912 )
13913 .await
13914 .expect("claim removal catalog.update succeeds");
13915 assert_eq!(
13916 forwarding.active_binding_count().unwrap(),
13917 1,
13918 "removing an attested target claim must leave the route census unchanged"
13919 );
13920 assert!(
13921 client_rx.try_recv().is_err(),
13922 "claim removal must not emit route.closed capability_denied"
13923 );
13924 assert!(
13925 target_rx.try_recv().is_err(),
13926 "claim removal must not send the target a route GOODBYE"
13927 );
13928 }
13929
13930 #[tokio::test]
13933 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
13934 let registry = Arc::new(Registry::default());
13935 let forwarding = Arc::new(ForwardingTable::default());
13936 let supervisor = SupervisorHandle::new();
13937 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
13938 let handler =
13939 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13940 .with_supervisor(supervisor);
13941 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
13942 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
13943 register_capability_manifest(
13944 &handler,
13945 &target_ctx,
13946 &mut target_rx,
13947 capability_manifest("target", &["credentials-provider/v1"], &[]),
13948 1,
13949 )
13950 .await;
13951 register_capability_manifest(
13952 &handler,
13953 &opener_ctx,
13954 &mut opener_rx,
13955 capability_manifest("opener", &[], &["credentials-provider/v1"]),
13956 2,
13957 )
13958 .await;
13959
13960 let (_client_rx, bind) = open_route_for_capability_test(
13961 &handler,
13962 &target_ctx,
13963 &mut target_rx,
13964 742,
13965 3,
13966 "target",
13967 None,
13968 )
13969 .await;
13970 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
13971 panic!("direct scope-honesty route must bind");
13972 };
13973 assert_eq!(principal, Some(Principal::Direct));
13974 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
13975 }
13976
13977 #[tokio::test]
13980 async fn must_never_reach_self_route_is_capability_forbidden() {
13981 let registry = Arc::new(Registry::default());
13982 let forwarding = Arc::new(ForwardingTable::default());
13983 let supervisor = SupervisorHandle::new();
13984 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
13985 let handler =
13986 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13987 .with_supervisor(supervisor);
13988 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
13989 register_capability_manifest(
13990 &handler,
13991 &self_ctx,
13992 &mut self_rx,
13993 capability_manifest(
13994 "self-provider",
13995 &["credentials-provider/v1"],
13996 &["credentials-provider/v1"],
13997 ),
13998 1,
13999 )
14000 .await;
14001
14002 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
14003 let replies = handler
14004 .handle_control_frame(
14005 &client_ctx,
14006 route_open_frame_with_admission_facts(
14007 2,
14008 "self-provider",
14009 unique_project_root("admission-facts"),
14010 Some(ConsumerIdentity {
14011 module_id: "self-provider".to_string(),
14012 launch_nonce: "self-nonce".to_string(),
14013 }),
14014 None,
14015 ),
14016 )
14017 .await
14018 .expect("self-route refusal returns a typed frame");
14019 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
14020 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
14021 assert!(
14022 self_rx.try_recv().is_err(),
14023 "self denial must not relay route.bind"
14024 );
14025 }
14026
14027 #[test]
14028 fn unsupported_channel_zero_frame_returns_error() {
14029 let handler = ControlHandler::default();
14030 let request = Frame::build(
14031 FrameType::Request,
14032 control_flags(),
14033 0,
14034 0,
14035 21,
14036 b"opaque".to_vec(),
14037 )
14038 .unwrap();
14039
14040 let response = handler
14041 .handle_control(ConnectionId::new(1), request)
14042 .unwrap();
14043
14044 assert_eq!(response[0].header.ty, FrameType::Error);
14045 assert_eq!(
14046 parse_error(&response[0])["code"],
14047 "unsupported_control_frame"
14048 );
14049 }
14050
14051 mod swap {
14056 use super::*;
14057
14058 const INCUMBENT: ConnectionId = ConnectionId::new(30);
14059 const CANDIDATE: ConnectionId = ConnectionId::new(40);
14060
14061 struct Swap {
14062 registry: Arc<Registry>,
14063 forwarding: Arc<ForwardingTable>,
14064 handler: ControlHandler,
14065 incumbent_ctx: RouteCtx,
14066 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14067 candidate_ctx: RouteCtx,
14068 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
14069 }
14070
14071 async fn swap_with_incumbent() -> Swap {
14072 let registry = Arc::new(Registry::default());
14073 let forwarding = Arc::new(ForwardingTable::default());
14074 let handler =
14075 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14076 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
14077 hello_via_sink(
14078 &handler,
14079 &incumbent_ctx,
14080 &mut incumbent_rx,
14081 hello_frame("aft", PROTOCOL_VERSION, 7),
14082 )
14083 .await;
14084 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
14085 Swap {
14086 registry,
14087 forwarding,
14088 handler,
14089 incumbent_ctx,
14090 incumbent_rx,
14091 candidate_ctx,
14092 candidate_rx,
14093 }
14094 }
14095
14096 fn register_candidate(swap: &Swap, ready: Option<bool>) {
14097 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
14098 candidate_manifest.ready = ready;
14099 let registration = swap
14100 .registry
14101 .register_candidate_with_control_ops(
14102 candidate_manifest,
14103 PROTOCOL_VERSION,
14104 CANDIDATE,
14105 module_baseline_control_ops(),
14106 )
14107 .unwrap();
14108 swap.forwarding
14109 .register_candidate_module_connection(
14110 CANDIDATE,
14111 "aft".to_string(),
14112 PROTOCOL_VERSION,
14113 manifest_concurrency(®istration.manifest),
14114 swap.candidate_ctx.egress.clone(),
14115 )
14116 .unwrap();
14117 }
14118
14119 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
14120 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
14121 swap.registry.promote_candidate("aft").unwrap().unwrap();
14122 cutover.incumbent.unwrap()
14123 }
14124
14125 #[tokio::test]
14134 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
14135 let mut swap = swap_with_incumbent().await;
14136 let hold = Arc::new(std::sync::Barrier::new(2));
14137 let handler = swap
14138 .handler
14139 .clone()
14140 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
14141 handler
14142 .supervisor
14143 .open_swap("aft", "candidate-nonce".to_string());
14144 let hello = {
14145 let handler = handler.clone();
14146 let candidate_ctx = swap.candidate_ctx.clone();
14147 std::thread::spawn(move || {
14148 handler
14149 .handle_hello(
14150 candidate_ctx.connection_id,
14151 Some(candidate_ctx.egress.clone()),
14152 hello_frame_with_nonce(
14153 "aft",
14154 PROTOCOL_VERSION,
14155 8,
14156 Some("candidate-nonce"),
14157 ),
14158 )
14159 .unwrap()
14160 })
14161 };
14162
14163 hold.wait();
14164 assert!(swap
14167 .registry
14168 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
14169 .unwrap()
14170 .is_some());
14171 assert!(swap
14172 .registry
14173 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14174 .unwrap()
14175 .is_none());
14176 assert!(swap
14177 .forwarding
14178 .module_endpoint_for_connection(CANDIDATE)
14179 .unwrap()
14180 .is_none());
14181
14182 hold.wait();
14183 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
14184 assert_eq!(
14185 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
14186 FrameType::HelloAck
14187 );
14188 let published = swap
14189 .registry
14190 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
14191 .unwrap()
14192 .expect("candidate is published once its HELLO completes");
14193 assert_eq!(published.connection_id, CANDIDATE);
14194 assert!(swap
14195 .forwarding
14196 .module_endpoint_for_connection(CANDIDATE)
14197 .unwrap()
14198 .is_some());
14199 }
14200
14201 fn keyed_total(counters: &Value, key: &str) -> u64 {
14202 counters[key]
14203 .as_object()
14204 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
14205 .unwrap_or(0)
14206 }
14207
14208 #[tokio::test]
14209 async fn replacement_logs_old_end_and_new_admission_once() {
14210 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
14211 let mut swap = swap_with_incumbent().await;
14212 swap.handler
14213 .supervisor
14214 .open_swap("aft", "candidate-nonce".to_string());
14215 hello_via_sink(
14216 &swap.handler,
14217 &swap.candidate_ctx,
14218 &mut swap.candidate_rx,
14219 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
14220 )
14221 .await;
14222 cutover(&swap);
14223 swap.handler.cleanup_connection(INCUMBENT).unwrap();
14224
14225 let captured = crate::router::test_log::captured_logs(&logs);
14226 println!("captured replacement registry logs:\n{captured}");
14227 let new_admissions: Vec<_> = captured
14228 .lines()
14229 .filter(|line| {
14230 line.contains("connection_id=40")
14231 && line.contains("swap candidate registered; not routable until cutover")
14232 })
14233 .collect();
14234 assert_eq!(
14235 new_admissions.len(),
14236 1,
14237 "unexpected admission logs: {captured}"
14238 );
14239 assert!(new_admissions[0].contains("module_id=aft"));
14240 assert!(new_admissions[0].contains("connection_id=40"));
14241 assert!(
14242 new_admissions[0].contains("swap candidate registered; not routable until cutover")
14243 );
14244 assert!(new_admissions[0].contains("ready=true"));
14245
14246 let old_admissions: Vec<_> = captured
14247 .lines()
14248 .filter(|line| {
14249 line.contains("module registered module_id=aft ")
14250 && line.contains("connection_id=30")
14251 })
14252 .collect();
14253 assert_eq!(
14254 old_admissions.len(),
14255 1,
14256 "unexpected incumbent admission: {captured}"
14257 );
14258
14259 let old_ends: Vec<_> = captured
14260 .lines()
14261 .filter(|line| {
14262 line.contains("connection_id=30") && line.contains("module registration ended")
14263 })
14264 .collect();
14265 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
14266 assert!(old_ends[0].contains("module_id=aft"));
14267 assert!(old_ends[0].contains("connection_id=30"));
14268 assert!(old_ends[0].contains("reason=replaced"));
14269 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
14270 assert_eq!(
14271 captured
14272 .lines()
14273 .filter(|line| line.contains("module registration promoted"))
14274 .count(),
14275 1,
14276 "unexpected promotion logs: {captured}"
14277 );
14278 }
14279
14280 #[tokio::test]
14286 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
14287 let mut swap = swap_with_incumbent().await;
14288 let handler = swap.handler.clone();
14289
14290 let cotenant = ConnectionId::new(31);
14292 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
14293 let (cotenant_task, cotenant_bind) = relay_route_open(
14294 &handler,
14295 cotenant,
14296 &cotenant_ctx.egress,
14297 &mut swap.incumbent_rx,
14298 100,
14299 "aft",
14300 "swap-cotenant",
14301 )
14302 .await;
14303 handler
14304 .handle_control_frame(
14305 &swap.incumbent_ctx,
14306 route_bind_ack(cotenant_bind.header.corr),
14307 )
14308 .await
14309 .unwrap();
14310 assert!(cotenant_task.await.unwrap().is_empty());
14311 let (cotenant_channel, cotenant_epoch) =
14312 published_route(&cotenant_rx.recv().await.unwrap());
14313
14314 let caller = ConnectionId::new(32);
14316 let (caller_ctx, mut caller_rx) = route_ctx(caller);
14317 let (caller_task, caller_bind) = relay_route_open(
14318 &handler,
14319 caller,
14320 &caller_ctx.egress,
14321 &mut swap.incumbent_rx,
14322 101,
14323 "aft",
14324 "swap-caller",
14325 )
14326 .await;
14327 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
14328
14329 register_candidate(&swap, None);
14330 cutover(&swap);
14331
14332 let ack = handler
14334 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
14335 .await;
14336 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
14337 if module_loop_error.is_some() {
14338 handler.cleanup_connection(INCUMBENT).unwrap();
14341 }
14342
14343 assert!(
14345 cotenant_rx.try_recv().is_err(),
14346 "the co-tenant route on the incumbent was torn down by one late ack: \
14347 {module_loop_error:?}"
14348 );
14349 assert!(matches!(
14350 swap.forwarding
14351 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
14352 .unwrap(),
14353 DataRoute::Client(DataRouteState::Bound(_))
14354 ));
14355 assert_eq!(module_loop_error, None);
14356 assert!(swap
14357 .registry
14358 .get_module_by_connection(INCUMBENT)
14359 .unwrap()
14360 .is_some());
14361
14362 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
14364 .await
14365 .expect("the incumbent is told to drop the abandoned binding")
14366 .unwrap()
14367 .frame;
14368 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
14369 assert_eq!(goodbye.header.channel, abandoned_channel);
14370 assert_eq!(goodbye.header.epoch, abandoned_epoch);
14371 assert!(swap.incumbent_rx.try_recv().is_err());
14372
14373 let response = caller_task.await.unwrap();
14375 assert_eq!(response.len(), 1);
14376 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14377 assert!(caller_rx.try_recv().is_err());
14378
14379 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
14383 let counters = handler.counters().snapshot();
14384 assert_eq!(
14385 keyed_total(&counters, "route_open_accepted_by_principal"),
14386 1
14387 );
14388 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
14389 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
14390 }
14391
14392 #[tokio::test]
14396 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
14397 let mut swap = swap_with_incumbent().await;
14398 register_candidate(&swap, None);
14399 let incumbent = cutover(&swap);
14400 swap.forwarding
14401 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
14402 .unwrap()
14403 .expect("the incumbent is still registered");
14404
14405 let client = ConnectionId::new(33);
14406 let (client_ctx, mut client_rx) = route_ctx(client);
14407 let route_handler = swap.handler.clone();
14408 let open_ctx = RouteCtx {
14409 connection_id: client,
14410 egress: client_ctx.egress.clone(),
14411 };
14412 let mut route_task = tokio::spawn(async move {
14413 route_handler
14414 .handle_control_frame(
14415 &open_ctx,
14416 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
14417 )
14418 .await
14419 .unwrap()
14420 });
14421 let bind = tokio::select! {
14422 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
14423 response = &mut route_task => {
14424 let response = response.unwrap();
14425 panic!(
14426 "post-cutover route.open was refused instead of relayed to the candidate: {}",
14427 parse_error(&response[0])["code"]
14428 );
14429 }
14430 };
14431 swap.handler
14432 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
14433 .await
14434 .unwrap();
14435 assert!(route_task.await.unwrap().is_empty());
14436 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
14437 match swap
14438 .forwarding
14439 .lookup_data_route(client, channel, epoch)
14440 .unwrap()
14441 {
14442 DataRoute::Client(DataRouteState::Bound(route)) => {
14443 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
14444 }
14445 other => panic!("expected a bound route on the candidate, got {other:?}"),
14446 }
14447 assert!(swap.incumbent_rx.try_recv().is_err());
14448 }
14449
14450 #[tokio::test]
14455 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
14456 let swap = swap_with_incumbent().await;
14457 register_candidate(&swap, Some(false));
14458 let update = Frame::build(
14459 FrameType::Request,
14460 control_flags(),
14461 0,
14462 0,
14463 55,
14464 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
14465 provides: manifest("aft", PROTOCOL_VERSION).provides,
14466 capabilities: None,
14467 ready: Some(true),
14468 })
14469 .unwrap(),
14470 )
14471 .unwrap();
14472
14473 let replies = swap
14474 .handler
14475 .handle_control_frame(&swap.candidate_ctx, update)
14476 .await
14477 .unwrap();
14478
14479 assert_eq!(replies.len(), 1);
14480 assert_eq!(
14481 replies[0].header.ty,
14482 FrameType::Response,
14483 "candidate catalog.update was refused: {:?}",
14484 serde_json::from_slice::<Value>(&replies[0].body).ok()
14485 );
14486 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
14487 assert_eq!(
14488 swap.registry
14489 .get_module("aft")
14490 .unwrap()
14491 .unwrap()
14492 .connection_id,
14493 INCUMBENT
14494 );
14495 }
14496 }
14497
14498 mod swap_admission {
14502 use super::*;
14503
14504 const INCUMBENT_NONCE: &str = "incumbent-nonce";
14505 const CANDIDATE_NONCE: &str = "candidate-nonce";
14506
14507 fn handler_with_incumbent(
14508 module_id: &str,
14509 reserved: bool,
14510 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
14511 let registry = Arc::new(Registry::default());
14512 let supervisor = SupervisorHandle::new();
14513 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
14514 if reserved {
14515 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
14516 }
14517 let handler =
14518 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
14519 let incumbent = handler
14520 .handle_control(
14521 ConnectionId::new(1),
14522 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
14523 )
14524 .unwrap();
14525 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
14526 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
14527 (registry, supervisor, handler)
14528 }
14529
14530 #[test]
14536 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
14537 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
14538
14539 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
14540 let replies = handler
14541 .handle_control(
14542 ConnectionId::new(connection),
14543 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
14544 )
14545 .unwrap();
14546 assert_eq!(replies[0].header.ty, FrameType::Error);
14547 assert_eq!(
14548 parse_error(&replies[0])["code"],
14549 "swap_token_invalid",
14550 "nonce {nonce:?}"
14551 );
14552 }
14553 assert!(registry.get_candidate("aft").unwrap().is_none());
14554 assert_eq!(
14555 registry.get_module("aft").unwrap().unwrap().connection_id,
14556 ConnectionId::new(1)
14557 );
14558
14559 let admitted = handler
14562 .handle_control(
14563 ConnectionId::new(4),
14564 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
14565 )
14566 .unwrap();
14567 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
14568 assert_eq!(
14569 registry
14570 .get_candidate("aft")
14571 .unwrap()
14572 .unwrap()
14573 .connection_id,
14574 ConnectionId::new(4)
14575 );
14576 assert_eq!(
14577 registry.get_module("aft").unwrap().unwrap().connection_id,
14578 ConnectionId::new(1),
14579 "the candidate must not take the active slot"
14580 );
14581 let replayed = handler
14582 .handle_control(
14583 ConnectionId::new(5),
14584 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
14585 )
14586 .unwrap();
14587 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
14588
14589 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
14593 let squatter = handler
14594 .handle_control(
14595 ConnectionId::new(6),
14596 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
14597 )
14598 .unwrap();
14599 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
14600 assert!(
14601 registry.get_module("aft").unwrap().is_none(),
14602 "a squatter took the active slot of an id being swapped"
14603 );
14604 }
14605
14606 #[test]
14611 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
14612 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
14613
14614 let replies = handler
14615 .handle_control(
14616 ConnectionId::new(2),
14617 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14618 )
14619 .unwrap();
14620
14621 assert_eq!(
14622 replies[0].header.ty,
14623 FrameType::HelloAck,
14624 "reserved candidate refused: {:?}",
14625 serde_json::from_slice::<Value>(&replies[0].body).ok()
14626 );
14627 assert_eq!(
14628 registry
14629 .get_candidate("vault")
14630 .unwrap()
14631 .unwrap()
14632 .connection_id,
14633 ConnectionId::new(2)
14634 );
14635 }
14636
14637 #[test]
14640 fn without_an_open_swap_the_ordinary_gates_decide() {
14641 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
14642 supervisor.close_swap("vault");
14643
14644 let candidate = handler
14645 .handle_control(
14646 ConnectionId::new(2),
14647 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
14648 )
14649 .unwrap();
14650 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
14651 let duplicate = handler
14652 .handle_control(
14653 ConnectionId::new(3),
14654 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
14655 )
14656 .unwrap();
14657 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14658 assert!(registry.get_candidate("vault").unwrap().is_none());
14659 }
14660 }
14661
14662 mod scopes {
14666 mod idle_tests {
14667 use super::*;
14668 include!("scope_idle_tests.rs");
14669 }
14670 mod route_open_timings {
14671 use super::*;
14672 include!("route_open_timing_tests.rs");
14673 }
14674 mod activity {
14675 include!("activity_scope_tests.rs");
14676 }
14677 mod delta {
14678 include!("scope_handler_delta_tests.rs");
14679 }
14680 mod call_modes {
14681 include!("scope_call_mode_tests.rs");
14682 }
14683
14684 use subc_protocol::scope::{
14685 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
14686 ScopeStatus,
14687 };
14688
14689 use super::*;
14690
14691 const OWNER: &str = "prefrontal-core";
14692
14693 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
14694 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
14695 }
14696
14697 async fn call(
14698 handler: &ControlHandler,
14699 ctx: &RouteCtx,
14700 request: &ModuleControlRequestFromModule,
14701 ) -> Frame {
14702 let body = serde_json::to_vec(request).unwrap();
14703 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
14704 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
14705 assert_eq!(replies.len(), 1, "{replies:?}");
14706 replies.pop().unwrap()
14707 }
14708
14709 async fn sync(
14710 handler: &ControlHandler,
14711 ctx: &RouteCtx,
14712 generation: u64,
14713 scopes: Vec<ScopeRecord>,
14714 ) -> Result<ModuleControlResponseToModule, String> {
14715 let reply = call(
14716 handler,
14717 ctx,
14718 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
14719 )
14720 .await;
14721 match reply.header.ty {
14722 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
14723 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
14724 }
14725 }
14726
14727 async fn describe(
14728 handler: &ControlHandler,
14729 ctx: &RouteCtx,
14730 owner: &str,
14731 scope_ref: &str,
14732 ) -> ModuleControlResponseToModule {
14733 let reply = call(
14734 handler,
14735 ctx,
14736 &ModuleControlRequestFromModule::ScopeDescribe {
14737 owner: Principal::Reserved {
14738 module_id: owner.to_string(),
14739 },
14740 scope_ref: scope_ref.to_string(),
14741 },
14742 )
14743 .await;
14744 assert_eq!(
14745 reply.header.ty,
14746 FrameType::Response,
14747 "{:?}",
14748 parse_error(&reply)
14749 );
14750 serde_json::from_slice(&reply.body).unwrap()
14751 }
14752
14753 #[tokio::test]
14754 async fn scope_describe_never_echoes_bind_proof_or_hash() {
14755 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
14756 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
14757 let mut rig = rig().await;
14758 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
14759 .await;
14760 let _bound = rig
14761 .bound(
14762 Some(AFT),
14763 PLEXUS,
14764 Some(
14765 rig.selector("s", Some(1))
14766 .with_bind_proof(Some(PROOF.into())),
14767 ),
14768 )
14769 .await;
14770 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
14771 let output = serde_json::to_string(&reply).unwrap();
14772 assert!(output.contains("\"status\":\"live\""));
14773 assert!(!output.contains(PROOF));
14774 assert!(!output.contains(HASH));
14775 }
14776
14777 async fn module(
14779 handler: &ControlHandler,
14780 connection: u64,
14781 module_id: &str,
14782 nonce: Option<&str>,
14783 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
14784 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
14785 hello_via_sink(
14786 handler,
14787 &ctx,
14788 &mut rx,
14789 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
14790 )
14791 .await;
14792 (ctx, rx)
14793 }
14794
14795 #[tokio::test]
14798 async fn a_client_connection_cannot_sync_or_describe() {
14799 let handler = ControlHandler::new(Arc::new(Registry::default()));
14800 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
14801 for request in [
14802 ModuleControlRequestFromModule::ScopeSync {
14803 generation: 1,
14804 scopes: vec![head("s", 1)],
14805 },
14806 ModuleControlRequestFromModule::ScopeDescribe {
14807 owner: Principal::Direct,
14808 scope_ref: "s".to_string(),
14809 },
14810 ] {
14811 let reply = call(&handler, &ctx, &request).await;
14812 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
14813 }
14814 assert!(
14815 !handler
14816 .scopes
14817 .read()
14818 .unwrap()
14819 .describe(
14820 &Principal::Reserved {
14821 module_id: OWNER.to_string()
14822 },
14823 "s"
14824 )
14825 .owner_synced
14826 );
14827 }
14828
14829 #[tokio::test]
14832 async fn a_module_without_a_supervised_launch_cannot_sync() {
14833 let handler = ControlHandler::new(Arc::new(Registry::default()));
14834 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
14835 assert_eq!(
14836 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
14837 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14838 );
14839 }
14840
14841 #[tokio::test]
14842 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
14843 let supervisor = SupervisorHandle::new();
14844 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14845 let handler = ControlHandler::new(Arc::new(Registry::default()))
14846 .with_supervisor(supervisor.clone());
14847 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14848 sync(&handler, &incumbent, 1, vec![head("s", 1)])
14849 .await
14850 .expect("the current launch syncs");
14851
14852 supervisor.open_swap(OWNER, "n2".to_string());
14855 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
14856 assert_eq!(
14857 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14858 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14859 );
14860 sync(&handler, &incumbent, 2, vec![head("s", 1)])
14861 .await
14862 .expect("the serving owner syncs during the swap");
14863
14864 supervisor.close_swap(OWNER);
14867 assert_eq!(
14868 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
14869 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14870 );
14871 sync(&handler, &incumbent, 3, vec![head("s", 1)])
14872 .await
14873 .expect("the serving owner syncs after the rollback");
14874 handler.cleanup_connection(candidate.connection_id).unwrap();
14875
14876 supervisor.open_swap(OWNER, "n3".to_string());
14881 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
14882 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
14883 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
14884 .await
14885 .expect("the promoted launch takes authority");
14886 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
14887 panic!("unexpected reply {reply:?}");
14888 };
14889 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
14890 assert_eq!(
14891 sync(&handler, &incumbent, 4, Vec::new()).await,
14892 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
14893 );
14894 }
14895
14896 #[tokio::test]
14899 async fn closing_the_authority_connection_frees_sync_authority() {
14900 let supervisor = SupervisorHandle::new();
14901 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14902 let handler = ControlHandler::new(Arc::new(Registry::default()))
14903 .with_supervisor(supervisor.clone());
14904 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
14905 sync(&handler, &first, 10, vec![head("s", 1)])
14906 .await
14907 .unwrap();
14908 handler.cleanup_connection(first.connection_id).unwrap();
14909
14910 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
14911 sync(&handler, &second, 1, vec![head("s", 1)])
14912 .await
14913 .expect("the next connection takes the released authority");
14914 }
14915
14916 #[tokio::test]
14917 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
14918 let supervisor = SupervisorHandle::new();
14919 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
14920 let handler =
14921 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
14922 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
14923 sync(&handler, &first, 10, vec![head("s", 1)])
14924 .await
14925 .unwrap();
14926 handler
14927 .handle_control_frame(
14928 &first,
14929 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
14930 )
14931 .await
14932 .unwrap();
14933 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
14934 sync(&handler, &second, 1, vec![head("s", 1)])
14935 .await
14936 .expect("GOODBYE releases authority even if the old socket remains open");
14937 }
14938
14939 #[tokio::test]
14940 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
14941 let registry = Arc::new(Registry::default());
14942 let supervisor_handle = SupervisorHandle::new();
14943 let supervisor =
14944 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
14945 .with_handle(supervisor_handle.clone())
14946 .with_daemon_incarnation("incarnation-7".to_string());
14947 supervisor
14950 .supervise_configured(
14951 ModuleSpec {
14952 module_id: OWNER.to_string(),
14953 program: PathBuf::from("/nonexistent/prefrontal-core"),
14954 args: Vec::new(),
14955 env: Vec::new(),
14956 reserved: false,
14957 reserved_prefixes: Vec::new(),
14958 protocol: ModuleProtocol::Subc,
14959 overlap: Default::default(),
14960 },
14961 false,
14962 )
14963 .unwrap();
14964 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
14965 let handler =
14966 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14967 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
14968
14969 let ModuleControlResponseToModule::ScopeDescribe {
14971 status,
14972 daemon_incarnation,
14973 owner_synced,
14974 owner_configured,
14975 scope,
14976 ..
14977 } = describe(&handler, &reader, OWNER, "s").await
14978 else {
14979 panic!("not a describe reply");
14980 };
14981 assert_eq!(status, ScopeStatus::NotLive);
14982 assert_eq!(daemon_incarnation, "incarnation-7");
14983 assert!(!owner_synced);
14984 assert!(owner_configured);
14985 assert!(scope.is_none());
14986
14987 let ModuleControlResponseToModule::ScopeDescribe {
14990 status,
14991 owner_configured,
14992 ..
14993 } = describe(&handler, &reader, "ghost", "s").await
14994 else {
14995 panic!("not a describe reply");
14996 };
14997 assert_eq!(status, ScopeStatus::NotLive);
14998 assert!(!owner_configured);
14999
15000 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
15002 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
15003 let ModuleControlResponseToModule::ScopeDescribe {
15004 status,
15005 scope_epoch,
15006 owner_synced,
15007 scope,
15008 ..
15009 } = describe(&handler, &reader, OWNER, "s").await
15010 else {
15011 panic!("not a describe reply");
15012 };
15013 assert_eq!(status, ScopeStatus::Live);
15014 assert_eq!(scope_epoch, Some(4));
15015 assert!(owner_synced);
15016 let stamp = scope.expect("a live scope carries its stamp");
15017 assert!(
15018 stamp.owner_authorized,
15019 "prefrontal-core is the default authority"
15020 );
15021 assert_eq!(stamp.kind, ScopeKind::Head);
15022 }
15023
15024 #[tokio::test]
15025 async fn scope_authority_owners_decides_owner_authorized() {
15026 let supervisor = SupervisorHandle::new();
15027 supervisor.set_spawn_nonce("broca", "b1".to_string());
15028 let handler = ControlHandler::new(Arc::new(Registry::default()))
15029 .with_supervisor(supervisor)
15030 .with_scope_authority_owners(vec!["broca".to_string()]);
15031 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
15032 let mut gated = head("s", 1);
15033 gated.attributes.agent_id = Some("agent".to_string());
15034 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
15035 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
15036 describe(&handler, &broca, "broca", "s").await
15037 else {
15038 panic!("not a describe reply");
15039 };
15040 assert!(scope.unwrap().owner_authorized);
15041 }
15042
15043 #[tokio::test]
15047 async fn scope_ops_and_the_scopes_capability_are_advertised() {
15048 let handler = ControlHandler::new(Arc::new(Registry::default()));
15049 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
15050 let ack = hello_via_sink(
15051 &handler,
15052 &ctx,
15053 &mut rx,
15054 hello_frame("m", PROTOCOL_VERSION, 1),
15055 )
15056 .await;
15057 let ack = parse_ack(&ack);
15058 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
15059 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
15060 }
15061 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
15062
15063 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
15064 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
15065 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
15066 let reply = handler
15067 .handle_control_frame(&client, frame)
15068 .await
15069 .unwrap()
15070 .pop()
15071 .unwrap();
15072 let ClientControlResponse::ServerDescribe { capabilities, .. } =
15073 serde_json::from_slice(&reply.body).unwrap()
15074 else {
15075 panic!("not a server.describe reply");
15076 };
15077 assert!(
15078 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
15079 "{capabilities:?}"
15080 );
15081 }
15082
15083 const PLEXUS: &str = "plexus";
15086 const OTHER: &str = "other";
15087 const AFT: &str = "aft";
15088 const BROCA: &str = "broca";
15089 const MAGIC: &str = "magic-context";
15090
15091 fn nonce(module_id: &str) -> String {
15092 format!("nonce-{module_id}")
15093 }
15094
15095 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
15096 let (tx, rx) = mpsc::channel(64);
15097 (
15098 RouteCtx {
15099 connection_id: ConnectionId::new(connection),
15100 egress: FrameSink::new(tx),
15101 },
15102 rx,
15103 )
15104 }
15105
15106 struct Rig {
15110 handler: ControlHandler,
15111 forwarding: Arc<ForwardingTable>,
15112 owner: RouteCtx,
15113 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15114 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
15115 generation: u64,
15116 next_connection: u64,
15117 _supervisor: Supervisor,
15118 }
15119
15120 async fn rig() -> Rig {
15121 rig_with_flow_support(true).await
15122 }
15123
15124 async fn rig_with_flow_support(flow_support: bool) -> Rig {
15125 rig_with_scope_capabilities(flow_support, false).await
15126 }
15127
15128 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
15129 let mut provides = Vec::new();
15130 if flow_support {
15131 provides.push("flow-scopes/v1");
15132 }
15133 if run_support {
15134 provides.push("agent-run-scopes/v1");
15135 }
15136 rig_with_target_capabilities(&provides).await
15137 }
15138
15139 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
15140 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
15141 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15142 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15145 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
15146 Frame::build(
15147 FrameType::Hello,
15148 control_flags(),
15149 0,
15150 0,
15151 connection,
15152 serde_json::to_vec(&body).unwrap(),
15153 )
15154 .unwrap()
15155 }
15156
15157 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
15158 let registry = Arc::new(Registry::default());
15159 let forwarding = Arc::new(ForwardingTable::default());
15160 let supervisor_handle = SupervisorHandle::new();
15161 let supervisor =
15162 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
15163 .with_handle(supervisor_handle.clone());
15164 supervisor
15165 .supervise_configured(
15166 ModuleSpec {
15167 module_id: OWNER.to_string(),
15168 program: PathBuf::from("/nonexistent/prefrontal-core"),
15169 args: Vec::new(),
15170 env: Vec::new(),
15171 reserved: false,
15172 reserved_prefixes: Vec::new(),
15173 protocol: ModuleProtocol::Subc,
15174 overlap: Default::default(),
15175 },
15176 false,
15177 )
15178 .unwrap();
15179 for module_id in [OWNER, AFT, BROCA, MAGIC] {
15180 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
15181 }
15182 let handler =
15183 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15184 .with_supervisor(supervisor_handle);
15185 let (owner, mut owner_rx) = wide_ctx(1);
15186 hello_via_sink(
15187 &handler,
15188 &owner,
15189 &mut owner_rx,
15190 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
15191 )
15192 .await;
15193 let mut modules = BTreeMap::new();
15194 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
15195 let (ctx, mut rx) = wide_ctx(connection);
15196 let hello = hello_with_capabilities(module_id, connection, provides);
15197 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
15198 modules.insert(module_id.to_string(), (ctx, rx));
15199 }
15200 Rig {
15201 handler,
15202 forwarding,
15203 owner,
15204 _owner_rx: owner_rx,
15205 modules,
15206 generation: 0,
15207 next_connection: 100,
15208 _supervisor: supervisor,
15209 }
15210 }
15211
15212 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
15213 ScopeCarrier::new(Principal::Reserved {
15214 module_id: module_id.to_string(),
15215 })
15216 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
15217 }
15218
15219 fn session(scope_epoch: u64) -> ScopeRecord {
15222 let mut record = head("s", scope_epoch);
15223 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
15224 record.attributes.agent_id = Some("agent-1".to_string());
15225 record.attributes.delegates = true;
15226 record
15227 }
15228
15229 impl Rig {
15230 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
15231 self.generation += 1;
15232 sync(&self.handler, &self.owner, self.generation, scopes)
15233 .await
15234 .expect("the owner's sync is accepted");
15235 }
15236
15237 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15238 ScopeSelector::new(
15239 Principal::Reserved {
15240 module_id: OWNER.to_string(),
15241 },
15242 scope_ref.to_string(),
15243 scope_epoch,
15244 )
15245 }
15246
15247 fn open_frame(
15248 &mut self,
15249 opener: Option<&str>,
15250 target: &str,
15251 scope: Option<ScopeSelector>,
15252 ) -> (
15253 RouteCtx,
15254 mpsc::Receiver<crate::router::OutboundFrame>,
15255 Frame,
15256 ) {
15257 self.next_connection += 1;
15258 let (ctx, rx) = wide_ctx(self.next_connection);
15259 let root = unique_project_root("scoped-open");
15260 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
15261 target: RouteTarget::ToolProvider {
15262 module_id: target.to_string(),
15263 },
15264 identity: BindIdentity::new(
15265 root.path().to_path_buf(),
15266 "unit".to_string(),
15267 "session".to_string(),
15268 ),
15269 consumer_identity: opener.map(|module_id| ConsumerIdentity {
15270 module_id: module_id.to_string(),
15271 launch_nonce: nonce(module_id),
15272 }),
15273 consumer_capabilities: None,
15274 role_versions: None,
15275 admission_facts: None,
15276 scope,
15277 })
15278 .unwrap();
15279 let frame = Frame::build(
15280 FrameType::Request,
15281 control_flags(),
15282 0,
15283 0,
15284 self.next_connection,
15285 body,
15286 )
15287 .unwrap();
15288 (ctx, rx, frame)
15289 }
15290
15291 async fn refused(
15293 &mut self,
15294 opener: Option<&str>,
15295 target: &str,
15296 scope: Option<ScopeSelector>,
15297 ) -> String {
15298 self.refusal_body(opener, target, scope).await["code"]
15299 .as_str()
15300 .unwrap()
15301 .to_string()
15302 }
15303
15304 async fn refusal_body(
15305 &mut self,
15306 opener: Option<&str>,
15307 target: &str,
15308 scope: Option<ScopeSelector>,
15309 ) -> Value {
15310 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
15311 let replies = tokio::time::timeout(
15312 Duration::from_secs(2),
15313 self.handler.handle_control_frame(&ctx, frame),
15314 )
15315 .await
15316 .expect("the open must be refused before waiting for a bind ack")
15317 .unwrap();
15318 assert_eq!(replies.len(), 1, "{replies:?}");
15319 assert_eq!(replies[0].header.ty, FrameType::Error);
15320 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15321 assert!(
15322 module_rx.try_recv().is_err(),
15323 "a refused open relays nothing"
15324 );
15325 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
15326 parse_error(&replies[0])
15327 }
15328
15329 async fn relayed(
15331 &mut self,
15332 opener: Option<&str>,
15333 target: &str,
15334 scope: Option<ScopeSelector>,
15335 ) -> Relayed {
15336 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
15337 let handler = self.handler.clone();
15338 let task_ctx = ctx.clone();
15339 let task = tokio::spawn(async move {
15340 handler
15341 .handle_control_frame(&task_ctx, frame)
15342 .await
15343 .unwrap()
15344 });
15345 let (_, module_rx) = self.modules.get_mut(target).unwrap();
15346 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
15347 .await
15348 .expect("the target receives the relayed route.bind")
15349 .unwrap()
15350 .frame;
15351 Relayed {
15352 target: target.to_string(),
15353 client: ctx,
15354 client_rx: rx,
15355 task,
15356 bind,
15357 }
15358 }
15359
15360 async fn ack(&self, relayed: &Relayed) {
15361 let (module, _) = &self.modules[&relayed.target];
15362 self.handler
15363 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
15364 .await
15365 .unwrap();
15366 }
15367
15368 async fn bound(
15370 &mut self,
15371 opener: Option<&str>,
15372 target: &str,
15373 scope: Option<ScopeSelector>,
15374 ) -> Bound {
15375 let relayed = self.relayed(opener, target, scope).await;
15376 self.ack(&relayed).await;
15377 let Relayed {
15378 target,
15379 client,
15380 mut client_rx,
15381 task,
15382 bind,
15383 } = relayed;
15384 assert!(
15385 task.await.unwrap().is_empty(),
15386 "the open is answered by commit"
15387 );
15388 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
15389 Bound {
15390 target,
15391 client,
15392 client_rx,
15393 channel,
15394 epoch,
15395 bind,
15396 }
15397 }
15398
15399 fn live(&self, route: &Bound) -> bool {
15400 matches!(
15401 self.forwarding
15402 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
15403 .unwrap(),
15404 DataRoute::Client(DataRouteState::Bound(_))
15405 )
15406 }
15407 }
15408
15409 struct Relayed {
15410 target: String,
15411 client: RouteCtx,
15412 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15413 task: tokio::task::JoinHandle<Vec<Frame>>,
15414 bind: Frame,
15415 }
15416
15417 struct Bound {
15418 target: String,
15419 client: RouteCtx,
15420 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15421 channel: u16,
15422 epoch: u32,
15423 bind: Frame,
15424 }
15425
15426 impl Bound {
15427 fn closed_reason(&mut self) -> RouteCloseReason {
15430 let mut reason = None;
15431 let mut goodbye = false;
15432 while let Ok(outbound) = self.client_rx.try_recv() {
15433 let frame = outbound.frame;
15434 match frame.header.ty {
15435 FrameType::Goodbye => {
15436 assert_eq!(
15437 (frame.header.channel, frame.header.epoch),
15438 (self.channel, self.epoch)
15439 );
15440 goodbye = true;
15441 }
15442 FrameType::Push => {
15443 let ClientControlPush::RouteClosed {
15444 reason: r,
15445 module_id,
15446 ..
15447 } = serde_json::from_slice(&frame.body).unwrap()
15448 else {
15449 panic!("unexpected push");
15450 };
15451 assert_eq!(module_id, self.target);
15452 reason = Some(r);
15453 }
15454 other => panic!("unexpected frame {other:?}"),
15455 }
15456 }
15457 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
15458 reason.expect("the client is told why the route closed")
15459 }
15460
15461 fn untouched(&mut self) -> bool {
15462 self.client_rx.try_recv().is_err()
15463 }
15464
15465 fn stamp(&self) -> Option<ScopeStamp> {
15466 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
15467 ModuleControlRequest::RouteBind { scope, .. } => scope,
15468 other => panic!("expected a route.bind, got {other:?}"),
15469 }
15470 }
15471 }
15472
15473 #[tokio::test]
15474 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
15475 ) {
15476 let mut rig = rig().await;
15477 let mut record = session(1);
15478 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
15479 record.child_owners = vec![Principal::Reserved {
15480 module_id: MAGIC.to_string(),
15481 }];
15482 rig.sync(vec![record]).await;
15483 let scope = || Some(rig_selector("s", Some(1)));
15484
15485 rig.bound(Some(OWNER), PLEXUS, scope()).await;
15488 rig.bound(Some(AFT), OTHER, scope()).await;
15489 rig.bound(Some(BROCA), PLEXUS, scope()).await;
15490
15491 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
15495 assert_eq!(
15496 rig.refused(opener, target, scope()).await,
15497 error_codes::SCOPE_NOT_CARRIER,
15498 "{opener:?} -> {target}"
15499 );
15500 }
15501 }
15502
15503 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
15504 ScopeSelector::new(
15505 Principal::Reserved {
15506 module_id: OWNER.to_string(),
15507 },
15508 scope_ref.to_string(),
15509 scope_epoch,
15510 )
15511 }
15512
15513 #[tokio::test]
15514 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
15515 let mut rig = rig().await;
15516 rig.sync(vec![session(1)]).await;
15517 for opener in [OWNER, AFT] {
15518 assert_eq!(
15519 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
15520 .await,
15521 error_codes::SCOPE_EPOCH_REQUIRED,
15522 "{opener}"
15523 );
15524 }
15525 }
15526
15527 #[tokio::test]
15528 async fn admission_separates_not_synced_not_live_and_ended() {
15529 let mut rig = rig().await;
15530 let code = rig
15532 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15533 .await;
15534 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
15535 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
15536
15537 let ghost = ScopeSelector::new(
15539 Principal::Reserved {
15540 module_id: "ghost".to_string(),
15541 },
15542 "s".to_string(),
15543 Some(1),
15544 );
15545 assert_eq!(
15546 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
15547 error_codes::SCOPE_NOT_LIVE
15548 );
15549
15550 rig.sync(vec![session(2)]).await;
15551 assert_eq!(
15552 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
15553 .await,
15554 error_codes::SCOPE_NOT_LIVE
15555 );
15556 for epoch in [1, 3] {
15557 assert_eq!(
15558 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
15559 .await,
15560 error_codes::SCOPE_ENDED,
15561 "epoch {epoch}"
15562 );
15563 }
15564 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
15566 .await;
15567 }
15568
15569 #[tokio::test]
15570 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
15571 let mut rig = rig().await;
15572 rig.sync(vec![session(1)]).await;
15573 let route = rig
15574 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15575 .await;
15576 let stamp = route.stamp().expect("a scoped bind carries the stamp");
15577 assert_eq!(stamp.scope_ref, "s");
15578 assert_eq!(stamp.scope_epoch, 1);
15579 assert_eq!(stamp.kind, ScopeKind::Head);
15580 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
15581 assert!(stamp.attributes.delegates);
15582 assert!(stamp.owner_authorized);
15583 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15584 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
15585
15586 let (broca, mut broca_rx) = wide_ctx(50);
15589 hello_via_sink(
15590 &rig.handler,
15591 &broca,
15592 &mut broca_rx,
15593 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
15594 )
15595 .await;
15596 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
15597 .await
15598 .unwrap();
15599 let own = ScopeSelector::new(
15600 Principal::Reserved {
15601 module_id: BROCA.to_string(),
15602 },
15603 "b".to_string(),
15604 Some(1),
15605 );
15606 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
15607 assert!(!route.stamp().unwrap().owner_authorized);
15608 }
15609
15610 #[tokio::test]
15611 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
15612 let mut rig = rig().await;
15613 let mut record = head("s", 1);
15614 record.carriers = vec![carrier(AFT, None)];
15615 let flow_id = "Flow:run-7/step_2!~";
15616 record.attributes.flow_id = Some(flow_id.to_string());
15617 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
15618 .await
15619 .unwrap();
15620 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
15621 panic!("not a sync reply");
15622 };
15623 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
15624 let route = rig
15625 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15626 .await;
15627 assert!(rig.live(&route), "the stamped bind committed");
15628 let stamp = route.stamp().expect("a flow scope carries a stamp");
15629 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
15630 assert_eq!(stamp.attributes.agent_id, None);
15631 assert!(!stamp.attributes.delegates);
15632 assert!(stamp.owner_authorized);
15633 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15634 assert_eq!(unscoped.stamp(), None);
15635 }
15636
15637 #[tokio::test]
15638 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
15639 let mut rig = rig_with_flow_support(false).await;
15640 let mut record = session(1);
15641 record.attributes.flow_id = Some("flow:7".to_string());
15642 rig.sync(vec![record]).await;
15643 for opener in [OWNER, AFT] {
15644 let body = rig
15645 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15646 .await;
15647 assert_eq!(body["code"], "target_flow_unsupported");
15648 let message = body["message"].as_str().unwrap();
15649 for required in [PLEXUS, "flow-scopes/v1"] {
15650 assert!(message.contains(required), "{message}");
15651 }
15652 }
15653 }
15654
15655 #[tokio::test]
15656 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
15657 let mut rig = rig_with_flow_support(true).await;
15658 let mut record = session(1);
15659 record.attributes.flow_id = Some("flow:7".to_string());
15660 rig.sync(vec![record]).await;
15661 for opener in [OWNER, AFT] {
15662 let route = rig
15663 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
15664 .await;
15665 assert!(rig.live(&route));
15666 assert_eq!(
15667 route.stamp().unwrap().attributes.flow_id.as_deref(),
15668 Some("flow:7")
15669 );
15670 }
15671 }
15672
15673 #[tokio::test]
15674 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
15675 use std::future::Future;
15676
15677 let mut rig = rig_with_flow_support(true).await;
15678 let mut record = session(1);
15679 record.attributes.flow_id = Some("flow:7".to_string());
15680 rig.sync(vec![record]).await;
15681 let (client, mut client_rx, frame) =
15682 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
15683 for _ in 0..64 {
15686 client.egress.try_send(route_bind_ack(1)).unwrap();
15687 }
15688 let handler = rig.handler.clone();
15689 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15690 std::future::poll_fn(|cx| {
15691 assert!(open.as_mut().poll(cx).is_pending());
15692 std::task::Poll::Ready(())
15693 })
15694 .await;
15695 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15696
15697 let old_connection = rig.modules[PLEXUS].0.connection_id;
15698 rig.handler.cleanup_connection(old_connection).unwrap();
15699 let (replacement, mut replacement_rx) = wide_ctx(200);
15700 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
15701 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
15702 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
15703 let hello = Frame::build(
15704 FrameType::Hello,
15705 control_flags(),
15706 0,
15707 0,
15708 200,
15709 serde_json::to_vec(&body).unwrap(),
15710 )
15711 .unwrap();
15712 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
15713
15714 client_rx.try_recv().unwrap();
15715 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15716 .await
15717 .expect("the replacement is refused without waiting for a bind ack")
15718 .unwrap();
15719 assert_eq!(replies.len(), 1);
15720 let body = parse_error(&replies[0]);
15721 assert_eq!(body["code"], "target_flow_unsupported");
15722 for required in [PLEXUS, "flow-scopes/v1"] {
15723 assert!(body["message"].as_str().unwrap().contains(required));
15724 }
15725 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
15726 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15727 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15728 assert!(rig
15729 .handler
15730 .registry
15731 .get_module_by_connection(replacement.connection_id)
15732 .unwrap()
15733 .is_some());
15734 }
15735
15736 #[tokio::test]
15737 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
15738 use std::future::Future;
15739 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
15740 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15741 const NEXT_HASH: &str =
15742 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
15743 let mut rig = rig().await;
15744 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
15745 record.carriers.push(ScopeCarrier::new(Principal::Direct));
15746 rig.sync(vec![record.clone()]).await;
15747 let (client, mut rx, frame) = rig.open_frame(
15748 None,
15749 PLEXUS,
15750 Some(
15751 rig.selector("s", Some(1))
15752 .with_bind_proof(Some(PROOF.into())),
15753 ),
15754 );
15755 for _ in 0..64 {
15758 client.egress.try_send(route_bind_ack(1)).unwrap();
15759 }
15760 let handler = rig.handler.clone();
15761 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15762 std::future::poll_fn(|cx| {
15763 assert!(open.as_mut().poll(cx).is_pending());
15764 std::task::Poll::Ready(())
15765 })
15766 .await;
15767 record.bind_proof_sha256 = Some(NEXT_HASH.into());
15768 rig.sync(vec![record]).await;
15769 rx.try_recv().unwrap();
15770 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15771 .await
15772 .unwrap()
15773 .unwrap();
15774 assert_eq!(
15775 parse_error(&replies[0])["code"],
15776 error_codes::SCOPE_BIND_PROOF_MISMATCH
15777 );
15778 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
15779 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
15780 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15781 }
15782
15783 #[tokio::test]
15784 async fn scope_bind_proof_reserved_openers_ignore_proof() {
15785 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15786 let mut rig = rig().await;
15787 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15788 .await;
15789 for opener in [OWNER, AFT] {
15790 for proof in [None, Some("malformed+proof=")] {
15791 let route = rig
15792 .bound(
15793 Some(opener),
15794 PLEXUS,
15795 Some(
15796 rig.selector("s", Some(1))
15797 .with_bind_proof(proof.map(str::to_string)),
15798 ),
15799 )
15800 .await;
15801 assert!(rig.live(&route));
15802 }
15803 }
15804 }
15805
15806 #[tokio::test]
15807 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
15808 use std::future::Future;
15809 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
15810 const NEXT_HASH: &str =
15811 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
15812 let mut rig = rig().await;
15813 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
15814 .await;
15815 let (client, mut rx, frame) =
15816 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
15817 for _ in 0..64 {
15818 client.egress.try_send(route_bind_ack(1)).unwrap();
15819 }
15820 let handler = rig.handler.clone();
15821 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
15822 std::future::poll_fn(|cx| {
15823 assert!(open.as_mut().poll(cx).is_pending());
15824 std::task::Poll::Ready(())
15825 })
15826 .await;
15827 rig.sync(vec![
15828 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
15829 ])
15830 .await;
15831 rx.try_recv().unwrap();
15832 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15833 let bind = std::future::poll_fn(|cx| {
15834 assert!(open.as_mut().poll(cx).is_pending());
15835 module_rx.poll_recv(cx)
15836 })
15837 .await
15838 .unwrap()
15839 .frame;
15840 handler
15841 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
15842 .await
15843 .unwrap();
15844 let replies = tokio::time::timeout(Duration::from_secs(2), open)
15845 .await
15846 .unwrap()
15847 .unwrap();
15848 assert_eq!(replies.len(), 1);
15849 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
15852 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15853 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
15854 let route = rig
15855 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
15856 .await;
15857 assert!(rig.live(&route));
15858 }
15859
15860 #[tokio::test]
15861 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
15862 {
15863 let mut rig = rig_with_flow_support(false).await;
15864 rig.sync(vec![session(1)]).await;
15865 let route = rig
15866 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15867 .await;
15868 assert!(rig.live(&route));
15869 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
15870 let mut record = session(1);
15871 record.attributes.flow_id = Some("flow:7".to_string());
15872 rig.sync(vec![record]).await;
15873 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
15874 assert!(rig.live(&unscoped));
15875 assert_eq!(unscoped.stamp(), None);
15876 }
15877
15878 #[tokio::test]
15879 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
15880 let mut rig = rig().await;
15881 let mut record = session(1);
15882 record.attributes.flow_id = Some("flow:7".to_string());
15883 rig.sync(vec![record.clone()]).await;
15884 let before = rig.forwarding.published_scope_tag(OWNER, "s");
15885 let mut owner_route = rig
15886 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15887 .await;
15888 let mut carrier_route = rig
15889 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
15890 .await;
15891 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
15892 rig.sync(vec![record.clone()]).await;
15893 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
15894 assert!(rig.live(&owner_route) && owner_route.untouched());
15895 assert!(rig.live(&carrier_route) && carrier_route.untouched());
15896
15897 record.attributes.flow_id = Some("flow:8".to_string());
15898 rig.sync(vec![record]).await;
15899 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
15900 let before = before.unwrap();
15901 assert_eq!(after.scope_epoch, before.scope_epoch);
15902 assert!(after.version > before.version);
15903 for route in [&mut owner_route, &mut carrier_route] {
15904 assert!(!rig.live(route));
15905 assert_eq!(
15906 route.closed_reason(),
15907 RouteCloseReason::ScopeDelegationChanged
15908 );
15909 }
15910 assert!(rig.live(&unscoped) && unscoped.untouched());
15911 for target in [PLEXUS, OTHER] {
15914 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
15915 let goodbye = module_rx
15916 .try_recv()
15917 .expect("the provider sees the drain")
15918 .frame;
15919 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15920 assert!(module_rx.try_recv().is_err());
15921 }
15922 let rebound = rig
15923 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15924 .await;
15925 assert_eq!(
15926 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
15927 Some("flow:8")
15928 );
15929 }
15930
15931 #[tokio::test]
15936 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
15937 let mut rig = rig().await;
15938 rig.sync(vec![session(1)]).await;
15939 let mut cotenant = rig
15940 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
15941 .await;
15942
15943 let mut changed = session(1);
15944 changed.child_owners.push(Principal::Reserved {
15945 module_id: MAGIC.to_string(),
15946 });
15947 let mut ended = None;
15948 for (code, next) in [
15949 (error_codes::SCOPE_CHANGED, vec![changed]),
15950 (error_codes::SCOPE_ENDED, Vec::new()),
15951 ] {
15952 let relayed = rig
15953 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
15954 .await;
15955 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
15956 ended = Some(next.is_empty());
15957 rig.sync(next).await;
15958 rig.ack(&relayed).await;
15959 let replies = relayed.task.await.unwrap();
15960 assert_eq!(replies.len(), 1, "{replies:?}");
15961 assert_eq!(parse_error(&replies[0])["code"], code);
15962 assert_eq!(
15963 subc_protocol::error_codes::is_retryable_route_open(code),
15964 code == error_codes::SCOPE_CHANGED
15965 );
15966 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
15967 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
15969 let mut goodbyes = Vec::new();
15972 while let Ok(outbound) = plexus_rx.try_recv() {
15973 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
15974 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
15975 }
15976 assert!(
15977 goodbyes.contains(&(bind_channel, bind_epoch)),
15978 "{goodbyes:?}"
15979 );
15980 assert!(rig
15981 .handler
15982 .registry
15983 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
15984 .unwrap()
15985 .is_some());
15986 }
15987 assert_eq!(ended, Some(true));
15988 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
15991 }
15992
15993 #[tokio::test]
15996 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
15997 struct Case {
15998 name: &'static str,
15999 change: fn(&mut ScopeRecord),
16000 closed: [Option<RouteCloseReason>; 4],
16003 }
16004 use RouteCloseReason::*;
16005 let cases = [
16006 Case {
16007 name: "a carrier entry removed",
16008 change: |r| {
16009 r.carriers.retain(|c| {
16010 c.principal
16011 != Principal::Reserved {
16012 module_id: AFT.to_string(),
16013 }
16014 })
16015 },
16016 closed: [None, Some(ScopeCarrierRemoved), None, None],
16017 },
16018 Case {
16019 name: "a target removed from a carrier",
16020 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
16021 closed: [None, None, None, Some(ScopeCarrierRemoved)],
16022 },
16023 Case {
16024 name: "a bare carrier narrowed to targets",
16025 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
16026 closed: [None, Some(ScopeCarrierRemoved), None, None],
16027 },
16028 Case {
16029 name: "delegates turned off",
16030 change: |r| r.attributes.delegates = false,
16031 closed: [Some(ScopeDelegationChanged); 4],
16032 },
16033 Case {
16034 name: "agent_id changed",
16035 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
16036 closed: [Some(ScopeDelegationChanged); 4],
16037 },
16038 Case {
16039 name: "a carrier added, child owners changed, the record re-sent",
16040 change: |r| {
16041 r.carriers.push(carrier(MAGIC, None));
16042 r.child_owners.push(Principal::Reserved {
16043 module_id: MAGIC.to_string(),
16044 });
16045 },
16046 closed: [None; 4],
16047 },
16048 Case {
16049 name: "a target added",
16050 change: |r| {
16051 r.carriers[1]
16052 .targets
16053 .as_mut()
16054 .unwrap()
16055 .push("third".to_string())
16056 },
16057 closed: [None; 4],
16058 },
16059 Case {
16060 name: "delegates turned on",
16061 change: |r| r.attributes.delegates = true,
16062 closed: [None; 4],
16063 },
16064 ];
16065 for case in cases {
16066 let mut rig = rig().await;
16067 rig.sync(vec![session(1)]).await;
16068 let scope = || Some(rig_selector("s", Some(1)));
16069 let mut routes = [
16070 rig.bound(Some(OWNER), PLEXUS, scope()).await,
16071 rig.bound(Some(AFT), PLEXUS, scope()).await,
16072 rig.bound(Some(BROCA), PLEXUS, scope()).await,
16073 rig.bound(Some(BROCA), OTHER, scope()).await,
16074 ];
16075 let mut record = session(1);
16076 (case.change)(&mut record);
16077 rig.sync(vec![record]).await;
16078 for (index, expected) in case.closed.iter().enumerate() {
16079 let route = &mut routes[index];
16080 match expected {
16081 Some(reason) => {
16082 assert!(!rig.live(route), "{}: route {index} still live", case.name);
16083 assert_eq!(
16084 route.closed_reason(),
16085 *reason,
16086 "{}: route {index}",
16087 case.name
16088 );
16089 }
16090 None => {
16091 assert!(rig.live(route), "{}: route {index} closed", case.name);
16092 assert!(
16093 route.untouched(),
16094 "{}: route {index} was told something",
16095 case.name
16096 );
16097 }
16098 }
16099 }
16100 }
16101 }
16102
16103 #[tokio::test]
16104 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
16105 for next in [Vec::new(), vec![session(2)]] {
16107 let mut rig = rig().await;
16108 rig.sync(vec![session(1)]).await;
16109 let mut route = rig
16110 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16111 .await;
16112 rig.sync(next).await;
16113 assert!(!rig.live(&route));
16114 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
16115 }
16116
16117 let mut rig = rig().await;
16120 let mut child = session(1);
16121 child.scope_ref = "child".to_string();
16122 child.kind = ScopeKind::Worker;
16123 child.parent = Some(ScopeParent::new(
16124 Principal::Reserved {
16125 module_id: OWNER.to_string(),
16126 },
16127 "s".to_string(),
16128 1,
16129 ));
16130 rig.sync(vec![session(1), child.clone()]).await;
16131 let mut child_route = rig
16132 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
16133 .await;
16134 assert_eq!(
16135 child_route.stamp().unwrap().parent_state,
16136 Some(ParentState::Linked)
16137 );
16138 rig.sync(vec![child]).await;
16139 assert!(!rig.live(&child_route));
16140 assert_eq!(
16141 child_route.closed_reason(),
16142 RouteCloseReason::ScopeParentEnded
16143 );
16144 }
16145
16146 #[tokio::test]
16147 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
16148 ) {
16149 let mut rig = rig().await;
16150 rig.sync(vec![session(1)]).await;
16151 let mut route = rig
16152 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16153 .await;
16154 let before = rig.forwarding.published_scope_tag(OWNER, "s");
16155 rig.sync(vec![session(1)]).await;
16156 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
16157 assert!(rig.live(&route) && route.untouched());
16158
16159 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
16163 .forwarding
16164 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
16165 .unwrap()
16166 else {
16167 panic!("the route is bound");
16168 };
16169 binding.flow.acquire_tagged(9, false).await.unwrap();
16170 let mut widened = session(1);
16171 widened.carriers.push(carrier(MAGIC, None));
16172 rig.sync(vec![widened]).await;
16173 assert!(rig.live(&route) && route.untouched());
16174 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16175 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
16176 assert_eq!(binding.flow.in_flight(), 1);
16179 binding
16180 .flow
16181 .acquire_tagged(10, false)
16182 .await
16183 .expect("the flow is still open");
16184 }
16185
16186 #[tokio::test]
16189 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
16190 let mut rig = rig().await;
16191 rig.sync(vec![session(1)]).await;
16192 let mut on_incumbent = rig
16193 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
16194 .await;
16195
16196 let (candidate, _candidate_rx) = wide_ctx(9);
16199 let registration = rig
16200 .handler
16201 .registry
16202 .register_candidate_with_control_ops(
16203 manifest(PLEXUS, PROTOCOL_VERSION),
16204 PROTOCOL_VERSION,
16205 candidate.connection_id,
16206 module_baseline_control_ops(),
16207 )
16208 .unwrap();
16209 rig.forwarding
16210 .register_candidate_module_connection(
16211 candidate.connection_id,
16212 PLEXUS.to_string(),
16213 PROTOCOL_VERSION,
16214 manifest_concurrency(®istration.manifest),
16215 candidate.egress.clone(),
16216 )
16217 .unwrap();
16218 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
16219 rig.handler
16220 .registry
16221 .promote_candidate(PLEXUS)
16222 .unwrap()
16223 .unwrap();
16224 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
16225
16226 rig.sync(Vec::new()).await;
16227 assert!(!rig.live(&on_incumbent));
16228 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
16229 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
16230 let goodbye = incumbent_rx
16231 .try_recv()
16232 .expect("the superseded endpoint is told")
16233 .frame;
16234 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16235 }
16236 }
16237}
16238
16239#[cfg(test)]
16240mod concurrency_default_exposure_tests {
16241 use super::*;
16242
16243 fn hello_body(role_json: &str) -> Vec<u8> {
16244 format!(
16245 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
16246 )
16247 .into_bytes()
16248 }
16249
16250 fn manifest_from(body: &[u8]) -> ModuleManifest {
16251 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
16252 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
16253 .expect("manifest parses")
16254 }
16255
16256 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
16257
16258 #[test]
16259 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
16260 let body = hello_body(&format!(
16261 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
16262 ));
16263 let manifest = manifest_from(&body);
16264 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16267 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
16268 }
16269
16270 #[test]
16271 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
16272 let body = hello_body(&format!(
16273 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
16274 ));
16275 let manifest = manifest_from(&body);
16276 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
16277 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16278 }
16279
16280 #[test]
16281 fn non_management_roles_are_never_reported() {
16282 let body = hello_body(
16283 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
16284 );
16285 let manifest = manifest_from(&body);
16286 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
16287 }
16288}