1use std::{
2 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
3 fmt,
4 net::SocketAddr,
5 path::{Path, PathBuf},
6 sync::{Arc, Mutex, RwLock},
7 time::{Duration, Instant as StdInstant},
8};
9
10use serde::{Deserialize, Serialize};
11use subc_control::{
12 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
13 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
14 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
15 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
16 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
17 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
18 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
19};
20use subc_protocol::{
21 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
22 error_codes,
23 manifest::{
24 validate_hello_capability_grammar, validate_hello_confirm_operations,
25 validate_hello_event_declarations, validate_hello_flow_operations,
26 validate_hello_rich_ui_descriptor, validate_hello_role_declarations,
27 validate_hello_self_signal_declarations, CapabilityDeclarations, CapabilityNeed,
28 Concurrency, ManifestProvenance, ModuleManifest, ProviderRole,
29 },
30 peer::{
31 encode_route_bind, ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY,
32 OPERATOR_APP_CAPABILITY,
33 },
34 scope::{
35 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
36 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
37 SCOPE_SYNC_OP,
38 },
39 session::{
40 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
41 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
42 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
43 ROLE_VERSIONS_FIELD,
44 },
45 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
46 Priority, RouteTarget, PROTOCOL_VERSION,
47};
48use tokio::time::{timeout_at, Instant};
49use tracing::{debug, info, warn};
50
51use crate::{
52 caller_identity::CallerIdentity,
53 capability_requirements::{
54 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
55 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
56 RequirementStatus, RuntimeModule,
57 },
58 child_roster::DaemonShutdownFlag,
59 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
60 forwarding::{
61 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
62 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
63 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
64 },
65 observability::{
66 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
67 },
68 privileged_prompt::{prompt_text, PrivilegedOperation},
69 provenance::{
70 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
71 },
72 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
73 router::{daemon_error_flags, RouteCtx, RouterError},
74 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
75 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
76 stderr_tail::{CaptureState, TailEntry},
77 supervise::{
78 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, SpawnSubscribeRefusal,
79 SupervisorHandle, SwapHelloAdmission,
80 },
81 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
82};
83
84pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
90
91const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
92const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
93const CAP_PING_PONG: &str = "ping_pong_v1";
94const CAP_SESSION_ATTACH: &str = "session_attach_v1";
95const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
96
97const SUBC_CONTROL_OPS: &[&str] = &[
98 ops::ACTIVITY_RECORD,
99 ops::ACTIVITY_QUERY,
100 ops::ACTIVITY_SUBSCRIBE,
101 ops::SERVER_DESCRIBE,
102 ops::CATALOG_LIST,
103 ops::ROUTE_OPEN,
104 ops::ROUTE_POLL,
105 ops::ROUTE_CLOSING,
106 ops::ROUTE_CLOSED,
107 ops::SUPERVISOR_LIST,
108 ops::SUPERVISOR_RESTART,
109 ops::SUPERVISOR_SWAP,
110 ops::SUPERVISOR_RELOAD,
111 ops::SUPERVISOR_RESCAN,
112 ops::SUPERVISOR_RELEASE_RESERVED,
113 ops::SUPERVISOR_SET_ENABLED,
114 ops::SUPERVISOR_HEALTH_PROBE,
115 ops::SUPERVISOR_HEALTH,
116 ops::SUPERVISOR_STDERR_TAIL,
117 ops::SUPERVISOR_TERMINALS,
118 ops::SUPERVISOR_ROUTES,
119 ops::SUPERVISOR_PROVENANCE,
120 ops::SUPERVISOR_SPAWN_SNAPSHOT,
121 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
122];
123
124const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
131
132mod rescan_checked;
133use rescan_checked::{RescanMode, RescanPlan};
134
135const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
136 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
137 "supervisor.live_roots",
138 SCOPE_SYNC_OP,
139 SCOPE_APPLY_OP,
140 SCOPE_DESCRIBE_OP,
141 "operator.confirm",
142];
143
144const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
148
149const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
150
151pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
159
160pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
175
176pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
194
195const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
196const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
197
198fn reload_verdict(
199 configured: &Path,
200 spawned_from: Option<&Path>,
201 image: subc_control::RunningImageAgreement,
202) -> PendingReloadVerdict {
203 let path = match spawned_from {
204 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
205 ReloadPathAgreement::Match
206 }
207 Some(spawned_from) => ReloadPathAgreement::Mismatch {
208 configured: configured.to_path_buf(),
209 spawned_from: spawned_from.to_path_buf(),
210 },
211 None => ReloadPathAgreement::Unavailable {
212 reason: if matches!(
213 image,
214 subc_control::RunningImageAgreement::Unavailable {
215 reason: subc_control::RunningImageUnavailableReason::NotRunning
216 }
217 ) {
218 ReloadPathUnavailableReason::NotRunning
219 } else {
220 ReloadPathUnavailableReason::SpawnedPathUnavailable
221 },
222 },
223 };
224 PendingReloadVerdict {
225 path,
226 image,
227 spec: None,
228 }
229}
230
231fn launch_spec_agreement(
232 configured: &crate::supervise::ModuleSpec,
233 running: bool,
234 spawned: Option<&crate::supervise::LaunchSpecDigest>,
235) -> subc_control::LaunchSpecAgreement {
236 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
237 if !running {
238 return Agreement::Unavailable {
239 reason: Reason::NotRunning,
240 };
241 }
242 match spawned {
243 None => Agreement::Unavailable {
244 reason: Reason::SpawnedSpecUnavailable,
245 },
246 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
247 Agreement::Match
248 }
249 Some(_) => Agreement::Mismatch,
250 }
251}
252
253fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
254 if configured == spawned_from {
255 return true;
256 }
257 #[cfg(windows)]
258 {
259 match (
265 std::fs::canonicalize(configured),
266 std::fs::canonicalize(spawned_from),
267 ) {
268 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
269 _ => false,
270 }
271 }
272 #[cfg(not(windows))]
273 {
274 false
275 }
276}
277
278#[derive(Clone)]
279struct DaemonProvenanceFacts {
280 build: DaemonBuildProvenance,
281 pid: Option<u32>,
282 started_at_ms: Option<u64>,
283 start_clock: Option<crate::clock::StartClock>,
284 executable_path: Option<PathBuf>,
285 executable_identity: Option<SpawnedFileIdentity>,
286 process_start_time: Option<u64>,
287 probe: ExecutableIdentityProbe,
288}
289
290impl Default for DaemonProvenanceFacts {
291 fn default() -> Self {
292 Self {
293 build: DaemonBuildProvenance {
294 build_git_sha: None,
295 build_lock_digest: None,
296 },
297 pid: None,
298 started_at_ms: None,
299 start_clock: None,
300 executable_path: None,
301 executable_identity: None,
302 process_start_time: None,
303 probe: ExecutableIdentityProbe::default(),
304 }
305 }
306}
307
308#[derive(Debug, Clone)]
309struct SupervisorRescanContext {
310 supervisor: Supervisor,
311 config_path: PathBuf,
312 configured_port: Option<u16>,
313 storage_config: Option<crate::daemon_config::StorageConfig>,
314 admission_facts_carrier_module_id: Option<String>,
315 admission_facts_targets: Option<Vec<String>>,
316 scope_authority_owners: Vec<String>,
317 privileged_operator_prompt: PrivilegedOperatorPrompt,
318 generation: Arc<std::sync::atomic::AtomicU64>,
324}
325
326const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
347 "reloading",
348 "supervisor_not_live",
349 "registration_not_active",
350 "no_forwarding_connection",
351 "relay_send_failed",
352];
353
354#[derive(Clone)]
356pub struct ControlHandler {
357 registry: Arc<Registry>,
358 activity: Option<crate::activity::ActivityLedger>,
359 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
360 forwarding: Arc<ForwardingTable>,
361 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
362 supervisor: SupervisorHandle,
363 subc_capabilities: Arc<[String]>,
364 route_bind_relay_timeout: Duration,
368 route_bind_relay_timeouts: BTreeMap<String, Duration>,
372 route_bind_breakers: RouteBindBreakers,
375 route_bind_concurrency: RouteBindConcurrency,
378 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
383 route_bind_breaker_threshold: u32,
385 route_bind_breaker_cooldown: Duration,
387 health_probe_timeout: Duration,
388 storage_config: Option<crate::daemon_config::StorageConfig>,
391 machine_id: Option<crate::machine_id::MachineId>,
395 admission_facts_carrier_module_id: Option<String>,
396 admission_facts_targets: Option<Vec<String>>,
397 scopes: Arc<RwLock<ScopeTable>>,
401 scope_deadline_added: Arc<tokio::sync::Notify>,
405 scope_authority_owners: Vec<String>,
408 privileged_operator_prompt: PrivilegedOperatorPrompt,
410 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
413 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
414 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
415 operator_app_policy: crate::operator_apps::OperatorAppPolicy,
416 operator_apps: Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
417 #[cfg(test)]
418 caller_lookup: Option<TestCallerLookup>,
419 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
422 rescan: Option<SupervisorRescanContext>,
423 connected_clients: ConnectedClients,
424 counters: DaemonCounters,
425 local_endpoint_unavailable: Option<String>,
428 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
429 daemon_shutdown: DaemonShutdownFlag,
434 daemon_provenance: DaemonProvenanceFacts,
435 #[cfg(test)]
436 control_dispatch_delay: Option<Duration>,
437 #[cfg(test)]
438 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
439 #[cfg(test)]
443 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
444 #[cfg(test)]
450 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
451}
452
453impl fmt::Debug for ControlHandler {
454 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
455 f.debug_struct("ControlHandler")
456 .field("registry", &self.registry)
457 .field("forwarding", &self.forwarding)
458 .field("process_liveness", &self.process_liveness.is_some())
459 .field("supervisor", &self.supervisor)
460 .field("subc_capabilities", &self.subc_capabilities)
461 .finish()
462 }
463}
464
465struct RouteOpenRequest {
466 target: RouteTarget,
467 identity: BindIdentity,
468 consumer_identity: Option<ConsumerIdentity>,
469 consumer_capabilities: Option<Vec<String>>,
470 role_versions: Option<BTreeMap<String, String>>,
471 admission_facts: Option<serde_json::Value>,
472 scope: Option<ScopeSelector>,
473}
474
475#[derive(Clone, Copy)]
476enum RouteOpenPhase {
477 Admission,
478 ProjectRoot,
479 EgressReservation,
480 BindSend,
481 BindAckWait,
482}
483
484struct RouteOpenTimings {
488 module_id: String,
489 principal: String,
490 admission: Duration,
491 project_root: Duration,
492 egress_reservation: Duration,
493 bind_send: Duration,
494 bind_ack_wait: Duration,
495 phase: Option<RouteOpenPhase>,
496 phase_started_at: Instant,
497}
498
499impl RouteOpenTimings {
500 fn new() -> Self {
501 Self {
502 module_id: String::new(),
503 principal: "unresolved".to_string(),
505 admission: Duration::ZERO,
506 project_root: Duration::ZERO,
507 egress_reservation: Duration::ZERO,
508 bind_send: Duration::ZERO,
509 bind_ack_wait: Duration::ZERO,
510 phase: Some(RouteOpenPhase::Admission),
511 phase_started_at: Instant::now(),
512 }
513 }
514
515 fn finish(&mut self) {
516 let elapsed = self.phase_started_at.elapsed();
517 match self.phase.take() {
518 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
519 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
520 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
521 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
522 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
523 None => {}
524 }
525 }
526
527 fn enter(&mut self, phase: RouteOpenPhase) {
528 self.finish();
529 self.phase = Some(phase);
530 self.phase_started_at = Instant::now();
531 }
532}
533
534struct RouteBindReservationGuard {
535 forwarding: Arc<ForwardingTable>,
536 endpoint: ModuleEndpointId,
537 relay_corr: u64,
538 armed: bool,
539}
540
541struct ModuleControlRpcGuard {
542 forwarding: Arc<ForwardingTable>,
543 endpoint: ModuleEndpointId,
544 corr: u64,
545 armed: bool,
546}
547
548impl ModuleControlRpcGuard {
549 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
550 Self {
551 forwarding,
552 endpoint,
553 corr,
554 armed: true,
555 }
556 }
557
558 fn disarm(&mut self) {
559 self.armed = false;
560 }
561}
562
563impl Drop for ModuleControlRpcGuard {
564 fn drop(&mut self) {
565 if self.armed {
566 let _ = self
567 .forwarding
568 .cancel_module_control_rpc(self.endpoint, self.corr);
569 }
570 }
571}
572
573impl RouteBindReservationGuard {
574 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
575 Self {
576 forwarding,
577 endpoint,
578 relay_corr,
579 armed: true,
580 }
581 }
582
583 fn release_and_disarm(&mut self) {
584 if !self.armed {
585 return;
586 }
587 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
588 self.endpoint,
589 self.relay_corr,
590 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
591 ) {
592 send_goodbye_target_best_effort(
593 &self.forwarding.counters(),
594 &target,
595 "canceled route.bind",
596 );
597 }
598 self.armed = false;
599 }
600
601 fn disarm(&mut self) {
602 self.armed = false;
603 }
604}
605
606impl Drop for RouteBindReservationGuard {
607 fn drop(&mut self) {
608 self.release_and_disarm();
609 }
610}
611
612#[derive(Debug, Clone, Default)]
642pub(crate) struct RouteBindBreakers {
643 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
644}
645
646#[derive(Debug, Clone, Default)]
647pub(crate) struct RouteBindConcurrency {
648 modules: Arc<Mutex<HashMap<String, usize>>>,
649}
650
651struct RouteBindConcurrencyGuard {
652 concurrency: RouteBindConcurrency,
653 module_id: String,
654}
655
656impl RouteBindConcurrency {
657 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
660 let mut modules = self
661 .modules
662 .lock()
663 .expect("route.bind concurrency mutex poisoned");
664 let in_flight = modules.entry(module_id.to_string()).or_default();
665 if *in_flight >= limit {
666 return Err(*in_flight);
667 }
668 *in_flight += 1;
669 Ok(RouteBindConcurrencyGuard {
670 concurrency: self.clone(),
671 module_id: module_id.to_string(),
672 })
673 }
674}
675
676impl Drop for RouteBindConcurrencyGuard {
677 fn drop(&mut self) {
678 let mut modules = self
679 .concurrency
680 .modules
681 .lock()
682 .expect("route.bind concurrency mutex poisoned");
683 let remove = {
684 let in_flight = modules
685 .get_mut(&self.module_id)
686 .expect("admitted route.bind has a concurrency entry");
687 *in_flight -= 1;
688 *in_flight == 0
689 };
690 if remove {
691 modules.remove(&self.module_id);
692 }
693 }
694}
695
696#[derive(Debug, Default)]
697struct ModuleBreakerState {
698 consecutive_timeouts: u32,
700 cooldown_until: Option<Instant>,
703 probe_in_flight: Option<Arc<()>>,
708}
709
710enum RouteBindAdmission<'a> {
712 Admitted {
713 guard: RouteBindBreakerGuard<'a>,
714 probe: bool,
717 },
718 Refused {
719 consecutive_timeouts: u32,
720 retry_in: Duration,
724 probe_in_flight: bool,
725 },
726}
727
728struct RouteBindBreakerGuard<'a> {
735 breakers: RouteBindBreakers,
736 module_id: &'a str,
737 probe_token: Option<Arc<()>>,
738 settled: bool,
739}
740
741impl RouteBindBreakerGuard<'_> {
742 fn record_accepted(&mut self) -> bool {
746 self.settled = true;
747 self.breakers.record_accepted(self.module_id)
748 }
749
750 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
753 self.settled = true;
754 self.breakers.record_timeout(
755 self.module_id,
756 self.probe_token.as_ref(),
757 threshold,
758 cooldown,
759 )
760 }
761
762 fn record_inconclusive(&mut self) {
771 self.settled = true;
772 self.breakers
773 .record_inconclusive(self.module_id, self.probe_token.as_ref());
774 }
775}
776
777impl Drop for RouteBindBreakerGuard<'_> {
778 fn drop(&mut self) {
779 if !self.settled {
780 self.breakers
781 .record_inconclusive(self.module_id, self.probe_token.as_ref());
782 }
783 }
784}
785
786struct BreakerOpened {
790 consecutive_timeouts: u32,
791 reopened_after_probe: bool,
794}
795
796impl RouteBindBreakers {
797 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
798 self.modules
799 .lock()
800 .expect("route.bind breaker mutex poisoned")
801 }
802
803 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
806 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
807 probe: probe_token.is_some(),
808 guard: RouteBindBreakerGuard {
809 breakers: self.clone(),
810 module_id,
811 probe_token,
812 settled: false,
813 },
814 };
815
816 let mut modules = self.lock();
817 let Some(state) = modules.get_mut(module_id) else {
818 return admitted(None);
819 };
820 let Some(cooldown_until) = state.cooldown_until else {
821 return admitted(None);
822 };
823 if state.probe_in_flight.is_some() {
824 return RouteBindAdmission::Refused {
825 consecutive_timeouts: state.consecutive_timeouts,
826 retry_in: Duration::ZERO,
827 probe_in_flight: true,
828 };
829 }
830 let now = Instant::now();
831 if now < cooldown_until {
832 return RouteBindAdmission::Refused {
833 consecutive_timeouts: state.consecutive_timeouts,
834 retry_in: cooldown_until - now,
835 probe_in_flight: false,
836 };
837 }
838 let token = Arc::new(());
839 state.probe_in_flight = Some(Arc::clone(&token));
840 admitted(Some(token))
841 }
842
843 fn record_accepted(&self, module_id: &str) -> bool {
844 self.lock()
845 .remove(module_id)
846 .is_some_and(|state| state.cooldown_until.is_some())
847 }
848
849 fn record_timeout(
850 &self,
851 module_id: &str,
852 probe_token: Option<&Arc<()>>,
853 threshold: u32,
854 cooldown: Duration,
855 ) -> Option<BreakerOpened> {
856 let mut modules = self.lock();
857 let state = modules.entry(module_id.to_string()).or_default();
858 let was_open = state.cooldown_until.is_some();
859 let was_probe = Self::owns_probe(state, probe_token);
860 if was_probe {
861 state.probe_in_flight = None;
862 }
863 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
864 if state.consecutive_timeouts < threshold {
865 return None;
866 }
867 state.cooldown_until = Some(Instant::now() + cooldown);
868 Some(BreakerOpened {
869 consecutive_timeouts: state.consecutive_timeouts,
870 reopened_after_probe: was_open && was_probe,
871 })
872 }
873
874 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
875 state
879 .probe_in_flight
880 .as_ref()
881 .zip(token)
882 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
883 }
884
885 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
886 if let Some(state) = self.lock().get_mut(module_id) {
887 if Self::owns_probe(state, probe_token) {
888 state.probe_in_flight = None;
889 }
890 }
891 }
892
893 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
908 self.lock()
909 .remove(module_id)
910 .map(|state| state.consecutive_timeouts)
911 .filter(|discarded| *discarded > 0)
912 }
913
914 fn open_snapshot(&self) -> Option<serde_json::Value> {
921 let now = Instant::now();
922 let modules = self.lock();
923 let open = modules
924 .iter()
925 .filter_map(|(module_id, state)| {
926 let cooldown_until = state.cooldown_until?;
927 Some((
928 module_id.clone(),
929 serde_json::json!({
930 "consecutive_timeouts": state.consecutive_timeouts,
931 "cooldown_remaining_ms":
932 cooldown_until.saturating_duration_since(now).as_millis() as u64,
933 "probe_in_flight": state.probe_in_flight.is_some(),
934 }),
935 ))
936 })
937 .collect::<serde_json::Map<String, serde_json::Value>>();
938 (!open.is_empty()).then_some(serde_json::Value::Object(open))
939 }
940}
941
942impl ControlHandler {
943 pub fn new(registry: Arc<Registry>) -> Self {
944 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
945 }
946
947 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
948 let counters = forwarding.counters();
949 let route_bind_breakers = forwarding.route_bind_breakers();
953 let route_bind_concurrency = forwarding.route_bind_concurrency();
954 let route_outages = forwarding.route_outages();
955 Self {
956 registry,
957 activity: None,
958 activity_streams: Arc::new(Mutex::new(HashMap::new())),
959 forwarding,
960 process_liveness: None,
961 supervisor: SupervisorHandle::new(),
962 subc_capabilities: Arc::from([
963 CAP_MANIFEST_REGISTRATION.to_string(),
964 CAP_CHANNEL_LIFECYCLE.to_string(),
965 CAP_PING_PONG.to_string(),
966 CAP_SESSION_ATTACH.to_string(),
967 CAP_ADMISSION_FACTS_RELAY.to_string(),
968 CAP_SCOPES_V1.to_string(),
969 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
970 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
971 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
972 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
973 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
974 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
975 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
976 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
977 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
978 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
979 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
980 ]),
981 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
982 route_bind_relay_timeouts: BTreeMap::new(),
983 route_bind_breakers,
984 route_bind_concurrency,
985 route_outages,
986 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
987 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
988 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
989 storage_config: None,
990 machine_id: None,
991 admission_facts_carrier_module_id: None,
992 admission_facts_targets: None,
993 scopes: Arc::new(RwLock::new(ScopeTable::new(
994 crate::daemon_config::default_scope_authority_owners(),
995 ))),
996 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
997 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
998 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
999 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1000 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1001 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1002 operator_app_policy: Default::default(),
1003 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1004 #[cfg(test)]
1005 caller_lookup: None,
1006 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1007 rescan: None,
1008 connected_clients: ConnectedClients::new(),
1009 local_endpoint_unavailable: None,
1010 counters,
1011 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1012 daemon_shutdown: DaemonShutdownFlag::default(),
1013 daemon_provenance: DaemonProvenanceFacts::default(),
1014 #[cfg(test)]
1015 control_dispatch_delay: None,
1016 #[cfg(test)]
1017 provenance_probe_override: None,
1018 #[cfg(test)]
1019 route_bind_relay_sent_hold: None,
1020 #[cfg(test)]
1021 hello_registry_to_forwarding_hold: None,
1022 }
1023 }
1024
1025 pub fn with_storage_config(
1028 mut self,
1029 storage_config: Option<crate::daemon_config::StorageConfig>,
1030 ) -> Self {
1031 self.storage_config = storage_config;
1032 self
1033 }
1034
1035 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1038 self.activity = Some(crate::activity::ActivityLedger::open(
1039 run_dir.join("activity.jsonl"),
1040 incarnation,
1041 ));
1042 self.supervisor.set_activity(self.activity.clone());
1043 self
1044 }
1045
1046 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1049 self.machine_id = machine_id;
1050 self
1051 }
1052
1053 pub fn with_admission_facts_config(
1057 mut self,
1058 carrier_module_id: Option<String>,
1059 targets: Option<Vec<String>>,
1060 ) -> Self {
1061 self.admission_facts_carrier_module_id = carrier_module_id;
1062 self.admission_facts_targets = targets;
1063 self
1064 }
1065
1066 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1070 self.scopes = Arc::new(RwLock::new(ScopeTable::new(owners.iter().cloned())));
1071 self.scope_authority_owners = owners;
1072 self
1073 }
1074
1075 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1077 self.privileged_operator_prompt = mode;
1078 self
1079 }
1080
1081 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1084 self.route_bind_relay_timeout = timeout;
1085 self
1086 }
1087
1088 pub fn with_route_bind_relay_timeouts(
1094 mut self,
1095 timeouts: impl IntoIterator<Item = (String, Duration)>,
1096 ) -> Self {
1097 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1098 self
1099 }
1100
1101 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1107 self.route_bind_relay_timeouts
1108 .get(module_id)
1109 .copied()
1110 .unwrap_or(self.route_bind_relay_timeout)
1111 }
1112
1113 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1121 self.route_bind_breaker_threshold = threshold.max(1);
1122 self.route_bind_breaker_cooldown = cooldown;
1123 self
1124 }
1125
1126 #[cfg(test)]
1127 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1128 self.health_probe_timeout = timeout;
1129 self
1130 }
1131
1132 #[cfg(test)]
1133 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1134 self.control_dispatch_delay = Some(delay);
1135 self
1136 }
1137
1138 #[cfg(test)]
1142 pub(crate) fn with_route_bind_relay_sent_hold(
1143 mut self,
1144 hold: Arc<tokio::sync::Barrier>,
1145 ) -> Self {
1146 self.route_bind_relay_sent_hold = Some(hold);
1147 self
1148 }
1149
1150 #[cfg(test)]
1155 pub(crate) fn with_hello_registry_to_forwarding_hold(
1156 mut self,
1157 hold: Arc<std::sync::Barrier>,
1158 ) -> Self {
1159 self.hello_registry_to_forwarding_hold = Some(hold);
1160 self
1161 }
1162
1163 #[cfg(test)]
1164 fn hold_between_registry_and_forwarding(&self) {
1165 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1166 hold.wait();
1167 hold.wait();
1168 }
1169 }
1170
1171 pub fn with_process_liveness(
1172 mut self,
1173 process_liveness: Arc<dyn ModuleProcessLiveness>,
1174 ) -> Self {
1175 self.process_liveness = Some(process_liveness);
1176 self
1177 }
1178
1179 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1180 self.supervisor = supervisor;
1181 self.supervisor.set_activity(self.activity.clone());
1182 self
1183 }
1184
1185 pub fn with_daemon_provenance(
1186 mut self,
1187 pid: u32,
1188 started_at_ms: u64,
1189 executable_path: Option<PathBuf>,
1190 build_git_sha: Option<String>,
1191 build_lock_digest: Option<String>,
1192 ) -> Self {
1193 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1194 let process_start_time = process_start_time(pid);
1195 self.daemon_provenance = DaemonProvenanceFacts {
1196 build: DaemonBuildProvenance {
1197 build_git_sha,
1198 build_lock_digest,
1199 },
1200 pid: Some(pid),
1201 started_at_ms: Some(started_at_ms),
1202 start_clock: None,
1203 executable_path,
1204 executable_identity,
1205 process_start_time,
1206 probe: ExecutableIdentityProbe::default(),
1207 };
1208 self
1209 }
1210
1211 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1212 self.daemon_provenance.start_clock = Some(clock);
1213 self
1214 }
1215
1216 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1218 self.scopes
1219 .write()
1220 .unwrap_or_else(|poisoned| poisoned.into_inner())
1221 .set_wall_clock(Arc::new(clock));
1222 self
1223 }
1224
1225 #[cfg(test)]
1226 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1227 self.provenance_probe_override = Some(result);
1228 self
1229 }
1230
1231 pub fn with_capability_config(
1235 self,
1236 modules: impl IntoIterator<Item = (String, bool)>,
1237 reserved_capabilities: BTreeMap<String, String>,
1238 ) -> Self {
1239 self.capability_evaluator
1240 .configure(modules, reserved_capabilities);
1241 self
1242 }
1243
1244 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1248 self.daemon_shutdown = flag;
1249 self
1250 }
1251
1252 pub fn with_supervisor_rescan(
1253 mut self,
1254 supervisor: Supervisor,
1255 config_path: impl Into<PathBuf>,
1256 configured_port: Option<u16>,
1257 ) -> Self {
1258 self.rescan = Some(SupervisorRescanContext {
1259 supervisor,
1260 config_path: config_path.into(),
1261 configured_port,
1262 storage_config: self.storage_config.clone(),
1263 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1264 admission_facts_targets: self.admission_facts_targets.clone(),
1265 scope_authority_owners: self.scope_authority_owners.clone(),
1266 privileged_operator_prompt: self.privileged_operator_prompt,
1267 generation: Arc::default(),
1268 });
1269 self
1270 }
1271
1272 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1273 self.connected_clients = connected_clients;
1274 self
1275 }
1276
1277 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1280 self.local_endpoint_unavailable = reason;
1281 self
1282 }
1283
1284 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1285 Arc::clone(&self.forwarding)
1286 }
1287
1288 pub(crate) fn counters(&self) -> DaemonCounters {
1289 self.counters.clone()
1290 }
1291
1292 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1295 tokio::spawn(async move {
1296 loop {
1297 self.capability_evaluator
1298 .wait_for_change_or_deadline()
1299 .await;
1300 self.refresh_capability_requirements();
1301 }
1302 });
1303 }
1304
1305 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1311 tokio::spawn(async move {
1312 loop {
1313 let has_deadlines = self
1314 .scopes
1315 .read()
1316 .unwrap_or_else(|poisoned| poisoned.into_inner())
1317 .has_deadlines();
1318 if !has_deadlines {
1319 self.scope_deadline_added.notified().await;
1320 continue;
1321 }
1322 tokio::time::sleep(Duration::from_secs(1)).await;
1323 if let Err(error) = self.sweep_expired_scopes() {
1324 warn!(%error, "scope expiry sweep failed");
1325 }
1326 }
1327 });
1328 }
1329
1330 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1331 let mut table = self
1332 .scopes
1333 .write()
1334 .unwrap_or_else(|poisoned| poisoned.into_inner());
1335 let swept = table.sweep_expired();
1336 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1337 drop(table);
1338 self.close_scope_drained_routes(drained);
1339 Ok(())
1340 }
1341
1342 fn publish_scope_changes(
1343 &self,
1344 changes: &[crate::scopes::ScopeTagChange],
1345 expired: &[crate::scopes::ScopeExpired],
1346 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1347 if changes.is_empty() {
1348 return Ok(Vec::new());
1349 }
1350 let drained = self
1351 .forwarding
1352 .publish_scope_changes(changes)
1353 .map_err(RouterError::Forwarding)?;
1354 let mut counts = HashMap::new();
1355 for route in &drained {
1356 *counts
1357 .entry((
1358 route.scope.owner.as_str(),
1359 route.scope.scope_ref.as_str(),
1360 route.scope.tag.scope_epoch,
1361 ))
1362 .or_insert(0usize) += 1;
1363 }
1364 for scope in expired {
1365 let routes_closed = counts
1366 .get(&(
1367 scope.owner.as_str(),
1368 scope.scope_ref.as_str(),
1369 scope.scope_epoch,
1370 ))
1371 .copied()
1372 .unwrap_or(0);
1373 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1374 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1375 routes_closed, "scope expired");
1376 }
1377 Ok(drained)
1378 }
1379
1380 fn runtime_capability_snapshot(
1381 &self,
1382 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1383 let runtime = self
1384 .supervisor
1385 .list()
1386 .into_iter()
1387 .map(|module| {
1388 let status = module.status().map_err(|err| {
1389 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1390 })?;
1391 Ok(RuntimeModule {
1392 module_id: status.module_id,
1393 state: status.state,
1394 enabled: status.enabled,
1395 })
1396 })
1397 .collect::<Result<Vec<_>, RouterError>>()?;
1398 let registrations = self.registry.list_registered_modules().map_err(|err| {
1404 RouterError::backend(
1405 0,
1406 0,
1407 format!("failed to list capability registrations: {err}"),
1408 )
1409 })?;
1410 let registrations = registrations
1411 .into_iter()
1412 .map(|registration| RegisteredModule {
1413 module_id: registration.manifest.module_id,
1414 module_version: registration.manifest.module_version,
1415 capabilities: registration.manifest.capabilities,
1416 })
1417 .collect();
1418 Ok((runtime, registrations))
1419 }
1420
1421 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1428 let cached_registration = RegisteredModule {
1429 module_id: registration.manifest.module_id.clone(),
1430 module_version: registration.manifest.module_version.clone(),
1431 capabilities: registration.manifest.capabilities.clone(),
1432 };
1433 let change = self.capability_evaluator.record_hello(&cached_registration);
1434 log_manifest_change(&cached_registration.module_id, &change);
1435 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1436 self.enforce_capability_denies();
1437 }
1438 self.refresh_capability_requirements();
1439 }
1440
1441 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1445 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1446 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1447 self.supervisor.set_swap_promotion_observer(observer);
1448 }
1449
1450 pub fn refresh_capability_requirements(&self) {
1451 match self.runtime_capability_snapshot() {
1452 Ok((runtime, registrations)) => {
1453 log_requirement_events(
1454 self.capability_evaluator
1455 .evaluate_now(&runtime, ®istrations),
1456 &self.daemon_shutdown,
1457 );
1458 }
1459 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1460 }
1461 }
1462
1463 fn enforce_capability_denies(&self) {
1467 let registrations = match self.registry.list_registered_modules() {
1468 Ok(snapshot) => snapshot,
1469 Err(err) => {
1470 warn!(error = %err, "failed to read registrations for capability deny census");
1471 return;
1472 }
1473 };
1474 let manifests = registrations
1475 .into_iter()
1476 .map(|registration| {
1477 (
1478 registration.manifest.module_id.clone(),
1479 registration.manifest,
1480 )
1481 })
1482 .collect::<BTreeMap<_, _>>();
1483 let census = match self.forwarding.route_census(None) {
1484 Ok(census) => census,
1485 Err(err) => {
1486 warn!(error = %err, "failed to read route census for capability deny enforcement");
1487 return;
1488 }
1489 };
1490
1491 for (target_module_id, routes) in census {
1492 let Some(target_manifest) = manifests.get(&target_module_id) else {
1493 continue;
1494 };
1495 let mut closed_routes = Vec::new();
1496 let mut goodbyes = Vec::new();
1497 for route in routes {
1498 let Principal::Reserved {
1499 module_id: opening_module_id,
1500 } = &route.principal
1501 else {
1502 continue;
1503 };
1504 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1505 continue;
1506 };
1507 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1508 continue;
1509 };
1510
1511 match self.forwarding.release_client_route(
1512 route.goodbye_target.connection_id,
1513 route.goodbye_target.channel,
1514 route.goodbye_target.epoch,
1515 ) {
1516 Ok(RouteRelease::Removed(module_goodbye)) => {
1517 warn!(
1518 opening_module_id,
1519 target_module_id,
1520 capability,
1521 "force-closing route because an attested capability deny edge now matches"
1522 );
1523 goodbyes.push(module_goodbye);
1524 goodbyes.push(route.goodbye_target.clone());
1525 closed_routes.push(route);
1526 }
1527 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1528 Err(err) => warn!(
1529 opening_module_id,
1530 target_module_id,
1531 capability,
1532 error = %err,
1533 "failed to force-close capability-denied route"
1534 ),
1535 }
1536 }
1537
1538 if closed_routes.is_empty() {
1539 continue;
1540 }
1541 send_route_control_pushes(
1542 &self.forwarding,
1543 closed_routes,
1544 ClientControlPush::RouteClosed {
1545 module_id: target_module_id,
1546 channels: Vec::new(),
1547 reason: RouteCloseReason::CapabilityDenied,
1548 drained: false,
1549 abandoned: 0,
1550 excluded_subscriptions: 0,
1551 terminal: Some(false),
1552 },
1553 );
1554 self.emit_route_goodbyes(goodbyes);
1557 }
1558 }
1559
1560 fn not_ready_reason(
1566 &self,
1567 registration: &crate::registry::ModuleRegistration,
1568 ) -> Option<NotReadyReason> {
1569 if !registration.ready {
1570 return Some(NotReadyReason {
1571 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1572 capability: None,
1573 });
1574 }
1575 self.first_unprovided_required_capability(registration)
1576 .map(|capability| NotReadyReason {
1577 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1578 capability: Some(capability),
1579 })
1580 }
1581
1582 fn first_unprovided_required_capability(
1600 &self,
1601 registration: &crate::registry::ModuleRegistration,
1602 ) -> Option<String> {
1603 let required = registration
1604 .manifest
1605 .capabilities
1606 .iter()
1607 .flat_map(|declarations| declarations.requires.iter())
1608 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1609 .map(|requirement| requirement.capability.as_str())
1610 .collect::<BTreeSet<_>>();
1611 if required.is_empty() {
1612 return None;
1613 }
1614 let module_id = registration.manifest.module_id.as_str();
1615 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1616 if required
1617 .iter()
1618 .any(|capability| verdict(capability).is_none())
1619 {
1620 self.refresh_capability_requirements();
1621 }
1622 required
1623 .into_iter()
1624 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1625 .map(str::to_string)
1626 }
1627
1628 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1629 self.capability_evaluator
1630 .statuses()
1631 .into_iter()
1632 .map(capability_requirement_status)
1633 .collect()
1634 }
1635
1636 fn deregister_connection(
1643 &self,
1644 connection_id: ConnectionId,
1645 reason: RegistrationEndReason,
1646 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1647 self.registry
1648 .deregister_connection_with_reason(connection_id, reason)
1649 }
1650
1651 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1652 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1653 return None;
1654 }
1655 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1656 parse_client_control_request(&frame.body)
1657 else {
1658 return None;
1659 };
1660 Some(target_module_id(&target).to_string())
1661 }
1662
1663 pub(crate) fn route_open_capacity_refusal(
1664 &self,
1665 ctx: &RouteCtx,
1666 frame: &Frame,
1667 target_module_id: &str,
1668 in_flight: usize,
1669 limit: usize,
1670 ) -> Result<Frame, RouterError> {
1671 let caller = self.route_open_caller_from_frame(frame);
1674 self.route_open_admission_refusal_frame(
1675 ctx,
1676 &caller,
1677 frame,
1678 target_module_id,
1679 "open_admission_full",
1680 (in_flight, limit),
1681 format!(
1682 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1683 ),
1684 )
1685 }
1686
1687 fn route_open_target_capacity_refusal(
1688 &self,
1689 ctx: &RouteCtx,
1690 caller: &RouteOpenCaller,
1691 frame: &Frame,
1692 target_module_id: &str,
1693 in_flight: usize,
1694 ) -> Result<Frame, RouterError> {
1695 self.route_open_admission_refusal_frame(
1696 ctx,
1697 caller,
1698 frame,
1699 target_module_id,
1700 "target_binds_full",
1701 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1702 format!(
1703 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1704 ),
1705 )
1706 }
1707
1708 #[allow(clippy::too_many_arguments)]
1716 fn route_open_admission_refusal_frame(
1717 &self,
1718 ctx: &RouteCtx,
1719 caller: &RouteOpenCaller,
1720 frame: &Frame,
1721 target_module_id: &str,
1722 reason: &'static str,
1723 (in_flight, limit): (usize, usize),
1724 message: impl Into<String>,
1725 ) -> Result<Frame, RouterError> {
1726 let code = error_codes::TARGET_UNAVAILABLE;
1727 self.counters.increment_route_open_refused(code);
1728 info!(
1729 target: "control",
1730 code,
1731 reason,
1732 module_id = ?target_module_id,
1733 connection_id = ctx.connection_id.get(),
1734 principal = %caller.principal,
1735 scope = %caller.scope,
1736 harness = %caller.harness,
1737 in_flight,
1738 limit,
1739 "route.open refused"
1740 );
1741 control_error_frame(frame, code, message.into())
1742 }
1743
1744 #[cfg(test)]
1750 pub fn handle_control(
1751 &self,
1752 connection_id: ConnectionId,
1753 frame: Frame,
1754 ) -> Result<Vec<Frame>, RouterError> {
1755 match frame.header.ty {
1756 FrameType::Ping => Ok(vec![pong(&frame)?]),
1757 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1758 FrameType::Goodbye => self.handle_goodbye(connection_id),
1759 ty => Ok(vec![control_error_frame(
1760 &frame,
1761 "unsupported_control_frame",
1762 format!("unsupported channel-0 frame {ty:?}"),
1763 )?]),
1764 }
1765 }
1766
1767 pub async fn handle_control_frame(
1768 &self,
1769 ctx: &RouteCtx,
1770 frame: Frame,
1771 ) -> Result<Vec<Frame>, RouterError> {
1772 self.handle_control_frame_timed(ctx, frame, None).await
1773 }
1774
1775 #[cfg(any(test, feature = "test-support"))]
1776 pub fn with_test_operator_app_policy(
1777 mut self,
1778 policy: crate::operator_apps::TestOperatorAppPolicy,
1779 ) -> Self {
1780 self.operator_app_policy = policy.into();
1781 self
1782 }
1783
1784 pub(crate) fn record_captured_peer(
1785 &self,
1786 connection: ConnectionId,
1787 peer: subc_os::local_peer::CapturedPeer,
1788 ) {
1789 self.record_observed_peer(connection, peer.observation().clone());
1790 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1791 self.operator_apps
1792 .lock()
1793 .unwrap_or_else(|p| p.into_inner())
1794 .insert(connection, stamp);
1795 }
1796 }
1797
1798 pub(crate) fn record_observed_peer(
1799 &self,
1800 connection: ConnectionId,
1801 peer: subc_os::local_peer::ObservedPeer,
1802 ) {
1803 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1804 let peer = match peer {
1805 OsPeer::Unobserved => ObservedPeer::unobserved(),
1806 OsPeer::Observed(peer) => {
1807 let observed = match peer.user {
1808 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1809 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1810 };
1811 observed
1812 .with_exe(
1813 peer.exe
1814 .and_then(|path| path.into_os_string().into_string().ok()),
1815 )
1816 .with_signing_id(peer.signing_id)
1817 }
1818 };
1819 self.observed_peers
1820 .lock()
1821 .unwrap_or_else(|p| p.into_inner())
1822 .insert(connection, peer);
1823 }
1824
1825 pub(crate) fn record_caller_endpoints(
1826 &self,
1827 connection: ConnectionId,
1828 endpoints: Option<(SocketAddr, SocketAddr)>,
1829 ) {
1830 let mut callers = self
1831 .caller_endpoints
1832 .lock()
1833 .unwrap_or_else(|p| p.into_inner());
1834 callers.remove(&connection);
1835 if let Some(endpoints) = endpoints {
1836 callers.insert(connection, endpoints);
1837 }
1838 }
1839
1840 #[cfg(test)]
1841 pub(crate) fn with_caller_lookup_for_test(
1842 mut self,
1843 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1844 ) -> Self {
1845 self.caller_lookup = Some(Arc::new(lookup));
1846 self
1847 }
1848
1849 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
1850 let endpoints = self
1851 .caller_endpoints
1852 .lock()
1853 .unwrap_or_else(|p| p.into_inner())
1854 .get(&connection)
1855 .copied();
1856 #[cfg(test)]
1857 let lookup = self.caller_lookup.clone();
1858 tokio::task::spawn_blocking(move || {
1859 #[cfg(test)]
1860 if let Some(lookup) = lookup {
1861 return lookup(endpoints);
1862 }
1863 endpoints
1864 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
1865 .unwrap_or_default()
1866 })
1867 .await
1868 .unwrap_or_default()
1869 }
1870
1871 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
1872 frame.header.channel == 0
1873 && frame.header.ty == FrameType::Request
1874 && self
1875 .forwarding
1876 .module_endpoint_for_connection(ctx.connection_id)
1877 .is_ok_and(|endpoint| endpoint.is_none())
1878 && parse_client_control_request(&frame.body)
1879 .ok()
1880 .is_some_and(|request| privileged_operation(&request).is_some())
1881 }
1882
1883 pub(crate) async fn handle_control_frame_timed(
1884 &self,
1885 ctx: &RouteCtx,
1886 frame: Frame,
1887 dispatch_started_at: Option<StdInstant>,
1888 ) -> Result<Vec<Frame>, RouterError> {
1889 match frame.header.ty {
1890 FrameType::Ping => Ok(vec![pong(&frame)?]),
1891 FrameType::Hello => {
1892 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
1893 }
1894 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
1895 FrameType::Cancel => {
1896 if self
1899 .forwarding
1900 .operator_confirms()
1901 .cancel(ctx.connection_id, frame.header.corr)
1902 || self
1903 .supervisor
1904 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
1905 || self.cancel_activity_subscription(ctx, &frame)?
1906 {
1907 Ok(Vec::new())
1908 } else {
1909 Ok(vec![control_error_frame(
1910 &frame,
1911 "unknown_subscription",
1912 "no subscription has this correlation id",
1913 )?])
1914 }
1915 }
1916 FrameType::Request => {
1917 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
1920 if op
1921 .as_ref()
1922 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
1923 {
1924 return self.handle_activity_record(ctx, frame).await;
1925 }
1926 if op.as_ref().is_some_and(|probe| {
1927 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
1928 }) {
1929 return self.handle_activity_read(ctx, frame).await;
1930 }
1931 if op
1932 .as_ref()
1933 .is_some_and(|probe| probe.op == "operator.confirm")
1934 {
1935 return self.handle_operator_confirm(ctx, frame);
1936 }
1937 if op
1938 .as_ref()
1939 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
1940 {
1941 return self.handle_scope_served_by(ctx.connection_id, frame);
1942 }
1943 if self
1944 .forwarding
1945 .module_endpoint_for_connection(ctx.connection_id)
1946 .map_err(RouterError::Forwarding)?
1947 .is_some()
1948 {
1949 if !is_known_module_request_op(&frame.body) {
1950 return Ok(vec![control_error_frame(
1951 &frame,
1952 "unsupported_control_frame",
1953 "module-originated channel-0 REQUEST is not supported",
1954 )?]);
1955 }
1956 let request = match parse_module_control_request_from_module(&frame.body) {
1957 Ok(request) => request,
1958 Err((err, ControlRequestBodyError::UnknownOp)) => {
1959 return Ok(vec![control_error_frame(
1960 &frame,
1961 "unsupported_control_frame",
1962 format!("unsupported module-originated channel-0 REQUEST: {err}"),
1963 )?])
1964 }
1965 Err((err, ControlRequestBodyError::InvalidBody)) => {
1966 return Ok(vec![control_error_frame(
1967 &frame,
1968 "invalid_control_body",
1969 format!("malformed module control body: {err}"),
1970 )?])
1971 }
1972 };
1973 let op = module_control_request_op(&request);
1974 let corr = frame.header.corr;
1975 log_control_dispatch_arrival(op, ctx.connection_id, corr);
1976 let result =
1977 self.handle_module_control_request(ctx.connection_id, frame, request);
1978 log_slow_control_dispatch(
1979 dispatch_started_at,
1980 op,
1981 ctx.connection_id,
1982 corr,
1983 None,
1984 );
1985 return result;
1986 }
1987
1988 if is_known_module_request_op(&frame.body) {
1989 return Ok(vec![control_error_frame(
1990 &frame,
1991 "not_registered",
1992 "catalog.update requires an active module registration owned by this connection",
1993 )?]);
1994 }
1995
1996 let request = match parse_client_control_request(&frame.body) {
1997 Ok(request) => request,
1998 Err((err, ControlRequestBodyError::UnknownOp)) => {
1999 return Ok(vec![control_error_frame(
2000 &frame,
2001 "unknown_control_op",
2002 format!("unknown client control op: {err}"),
2003 )?])
2004 }
2005 Err((err, ControlRequestBodyError::InvalidBody)) => {
2006 return Ok(vec![control_error_frame(
2007 &frame,
2008 "invalid_control_body",
2009 format!("malformed client control body: {err}"),
2010 )?])
2011 }
2012 };
2013 let op = client_control_request_op(&request);
2014 let corr = frame.header.corr;
2015 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2016 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2017 #[cfg(test)]
2018 if let Some(delay) = self.control_dispatch_delay {
2019 tokio::time::sleep(delay).await;
2020 }
2021 let result = self
2022 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2023 .await;
2024 if let Some(timings) = route_open_timings.as_mut() {
2025 timings.finish();
2026 }
2027 log_slow_control_dispatch(
2028 dispatch_started_at,
2029 op,
2030 ctx.connection_id,
2031 corr,
2032 route_open_timings.as_ref(),
2033 );
2034 result
2035 }
2036 FrameType::Push => {
2037 let Some(endpoint) = self
2038 .forwarding
2039 .module_endpoint_for_connection(ctx.connection_id)
2040 .map_err(RouterError::Forwarding)?
2041 else {
2042 return Ok(vec![control_error_frame(
2043 &frame,
2044 "unsupported_control_frame",
2045 "client-originated channel-0 PUSH is not supported",
2046 )?]);
2047 };
2048 self.handle_status_update(endpoint, frame)
2049 }
2050 FrameType::Response | FrameType::Error
2051 if self
2052 .forwarding
2053 .module_endpoint_for_connection(ctx.connection_id)
2054 .map_err(RouterError::Forwarding)?
2055 .is_some() =>
2056 {
2057 self.handle_module_relay_response(ctx.connection_id, frame)
2058 }
2059 ty => Ok(vec![control_error_frame(
2060 &frame,
2061 "unsupported_control_frame",
2062 format!("unsupported channel-0 frame {ty:?}"),
2063 )?]),
2064 }
2065 }
2066
2067 pub fn cleanup_connection(
2068 &self,
2069 connection_id: ConnectionId,
2070 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2071 self.caller_endpoints
2072 .lock()
2073 .unwrap_or_else(|p| p.into_inner())
2074 .remove(&connection_id);
2075 self.build_class_warned
2076 .lock()
2077 .unwrap_or_else(|p| p.into_inner())
2078 .remove(&connection_id);
2079 self.observed_peers
2080 .lock()
2081 .unwrap_or_else(|p| p.into_inner())
2082 .remove(&connection_id);
2083 self.operator_apps
2084 .lock()
2085 .unwrap_or_else(|p| p.into_inner())
2086 .remove(&connection_id);
2087 self.forwarding
2088 .operator_confirms()
2089 .connection_closed(connection_id);
2090 self.cleanup_connection_with_end_reason(
2091 connection_id,
2092 RegistrationEndReason::ConnectionClosed,
2093 )
2094 }
2095
2096 fn cleanup_connection_with_end_reason(
2097 &self,
2098 connection_id: ConnectionId,
2099 requested_reason: RegistrationEndReason,
2100 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2101 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2102 self.registry
2103 .get_module_by_connection(connection_id)?
2104 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2105 .and_then(|module| module.registration_end_reason().ok().flatten())
2106 .unwrap_or(requested_reason)
2107 } else {
2108 requested_reason
2109 };
2110 let crash_closed = self
2111 .registry
2112 .get_module_by_connection(connection_id)?
2113 .and_then(|registration| {
2114 self.forwarding
2115 .module_endpoint_for_connection(connection_id)
2116 .ok()
2117 .flatten()
2118 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2119 .map(|routes| (registration.manifest.module_id, routes))
2120 });
2121 let crash_closed = crash_closed.map(|(module_id, routes)| {
2122 let terminal = match self.supervisor.get(&module_id) {
2123 None => false,
2124 Some(module) => match module.will_recover_after_connection_loss() {
2125 Ok(will_recover) => !will_recover,
2126 Err(err) => {
2127 warn!(
2128 %module_id,
2129 error = %err,
2130 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2131 );
2132 false
2133 }
2134 },
2135 };
2136 let (reason, drained) = match self
2140 .forwarding
2141 .daemon_drain_result_for_connection(connection_id)
2142 {
2143 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2144 Ok(None) => (RouteCloseReason::Crash, false),
2145 Err(err) => {
2146 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2147 (RouteCloseReason::Crash, false)
2148 }
2149 };
2150 (module_id, routes, reason, drained, terminal)
2151 });
2152 let registrations = self.deregister_connection(connection_id, end_reason);
2153 let cleanup = if crash_closed.is_some() {
2154 self.forwarding.cleanup_connection_counted(connection_id)
2155 } else {
2156 self.forwarding
2162 .cleanup_connection(connection_id)
2163 .map(|released| crate::forwarding::ConnectionCleanup {
2164 released,
2165 abandoned_relays: 0,
2166 })
2167 };
2168 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2173 let abandoned = cleanup
2174 .as_ref()
2175 .map_or(0, |cleanup| cleanup.abandoned_relays);
2176 send_route_control_pushes(
2177 &self.forwarding,
2178 routes,
2179 ClientControlPush::RouteClosed {
2180 module_id,
2181 channels: Vec::new(),
2182 reason,
2183 drained,
2184 abandoned,
2185 excluded_subscriptions: 0,
2186 terminal: Some(terminal),
2187 },
2188 );
2189 }
2190 if let Ok(cleanup) = cleanup {
2191 self.emit_route_goodbyes(cleanup.released);
2192 }
2193 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2197 crate::supervise::notify_registration_release();
2198 self.capability_evaluator.wake_deadline_loop();
2199 self.refresh_capability_requirements();
2200 }
2201 self.supervisor.remove_spawn_subscribers(connection_id);
2202 self.activity_streams
2203 .lock()
2204 .unwrap_or_else(|p| p.into_inner())
2205 .retain(|(id, _), task| {
2206 if *id == connection_id {
2207 task.abort();
2208 false
2209 } else {
2210 true
2211 }
2212 });
2213 self.hello_launch_nonces
2216 .lock()
2217 .unwrap_or_else(|poisoned| poisoned.into_inner())
2218 .forget(connection_id);
2219 self.scopes
2220 .write()
2221 .unwrap_or_else(|poisoned| poisoned.into_inner())
2222 .release_connection(connection_id);
2223 registrations
2224 }
2225
2226 pub(crate) fn handle_route_goodbye(
2227 &self,
2228 connection_id: ConnectionId,
2229 route_channel: u16,
2230 route_epoch: u32,
2231 ) -> Result<bool, RouterError> {
2232 debug!(
2233 connection_id = connection_id.get(),
2234 route_channel, route_epoch, "handling route GOODBYE"
2235 );
2236 let RouteRelease::Removed(released_route) = self
2237 .forwarding
2238 .release_client_route(connection_id, route_channel, route_epoch)
2239 .map_err(RouterError::Forwarding)?
2240 else {
2241 return Ok(false);
2242 };
2243 self.emit_route_goodbyes(vec![released_route]);
2244 Ok(true)
2245 }
2246
2247 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2248 for released in released_routes {
2249 let frame = match Frame::build_with_version(
2250 released.negotiated_ver,
2251 FrameType::Goodbye,
2252 control_flags(),
2253 released.channel,
2254 released.epoch,
2255 0,
2256 Vec::new(),
2257 ) {
2258 Ok(frame) => frame,
2259 Err(err) => {
2260 warn!(
2261 route_channel = released.channel,
2262 error = %err,
2263 "failed to build route GOODBYE frame"
2264 );
2265 continue;
2266 }
2267 };
2268 if !released.close_on_delivery_failure() {
2269 crate::forwarding::send_module_route_goodbye(
2270 &self.counters,
2271 &released.sink,
2272 frame,
2273 released.module_id.as_deref(),
2274 "client route released",
2275 );
2276 continue;
2277 }
2278 if let Err(err) = released.sink.try_send(frame) {
2279 warn!(
2280 target_connection_id = released.connection_id.get(),
2281 route_channel = released.channel,
2282 error = %err,
2283 "route GOODBYE was not delivered to client; closing target connection"
2284 );
2285 if self
2286 .forwarding
2287 .escalate_client_delivery_failure(
2288 released.connection_id,
2289 released.channel,
2290 released.epoch,
2291 CloseReason::new(
2292 "route_goodbye_delivery_failed",
2293 format!(
2294 "failed to enqueue route GOODBYE for channel {}: {err}",
2295 released.channel
2296 ),
2297 ),
2298 crate::forwarding::UndeliveredFrame {
2299 module_id: released.module_id.as_deref(),
2300 sink: &released.sink,
2301 },
2302 )
2303 .unwrap_or(false)
2304 {
2305 self.counters.increment_goodbye_relay_client_failed();
2306 }
2307 }
2308 }
2309 }
2310
2311 fn send_abandoned_route_bind_goodbye(
2323 &self,
2324 module_sink: &crate::FrameSink,
2325 negotiated_ver: u8,
2326 module_channel: u16,
2327 module_epoch: u32,
2328 ) {
2329 let frame = match Frame::build_with_version(
2330 negotiated_ver,
2331 FrameType::Goodbye,
2332 control_flags(),
2333 module_channel,
2334 module_epoch,
2335 0,
2336 Vec::new(),
2337 ) {
2338 Ok(frame) => frame,
2339 Err(err) => {
2340 warn!(
2341 route_channel = module_channel,
2342 error = %err,
2343 "failed to build GOODBYE for abandoned route.bind"
2344 );
2345 return;
2346 }
2347 };
2348 crate::forwarding::send_module_route_goodbye(
2349 &self.counters,
2350 module_sink,
2351 frame,
2352 None,
2353 "abandoned route.bind",
2354 );
2355 }
2356
2357 fn handle_hello(
2358 &self,
2359 connection_id: ConnectionId,
2360 sink: Option<crate::FrameSink>,
2361 frame: Frame,
2362 ) -> Result<Vec<Frame>, RouterError> {
2363 debug!(
2364 connection_id = connection_id.get(),
2365 corr = frame.header.corr,
2366 "handling HELLO"
2367 );
2368 if self
2372 .registry
2373 .get_module_by_connection(connection_id)
2374 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2375 .is_some()
2376 {
2377 return Ok(vec![control_error_frame(
2378 &frame,
2379 "invalid_hello",
2380 "connection is already registered as a module",
2381 )?]);
2382 }
2383 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2384 Ok(value) => value,
2385 Err(err) => {
2386 return Ok(vec![control_error_frame(
2387 &frame,
2388 "invalid_hello",
2389 format!("malformed HELLO body: {err}"),
2390 )?])
2391 }
2392 };
2393 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2394 return Ok(vec![control_error_frame(
2395 &frame,
2396 "invalid_capability_grammar",
2397 err.to_string(),
2398 )?]);
2399 }
2400 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2401 return Ok(vec![control_error_frame(
2402 &frame,
2403 "invalid_manifest",
2404 err.to_string(),
2405 )?]);
2406 }
2407 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2408 return Ok(vec![control_error_frame(
2409 &frame,
2410 "invalid_event_declaration",
2411 err.to_string(),
2412 )?]);
2413 }
2414 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2415 return Ok(vec![control_error_frame(
2416 &frame,
2417 "invalid_manifest",
2418 err.to_string(),
2419 )?]);
2420 }
2421 if let Some(provenance) = hello_value
2422 .get("manifest")
2423 .and_then(|manifest| manifest.get("provenance"))
2424 {
2425 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2426 return Ok(vec![control_error_frame(
2427 &frame,
2428 "invalid_manifest",
2429 format!("malformed manifest provenance: {err}"),
2430 )?]);
2431 }
2432 }
2433 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2434 let module_id = hello_value["manifest"]["module_id"]
2435 .as_str()
2436 .unwrap_or("<unknown>");
2437 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2438 }
2439 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2440 let module_id = hello_value["manifest"]["module_id"]
2441 .as_str()
2442 .unwrap_or("<unknown>");
2443 warn!(module_id, %reason, "dropping flow_operations from HELLO");
2444 }
2445 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2446 let module_id = hello_value["manifest"]["module_id"]
2447 .as_str()
2448 .unwrap_or("<unknown>");
2449 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2450 }
2451 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2452 Ok(hello) => hello,
2453 Err(err) => {
2454 return Ok(vec![control_error_frame(
2455 &frame,
2456 "invalid_hello",
2457 format!("malformed HELLO body: {err}"),
2458 )?])
2459 }
2460 };
2461
2462 if hello.protocol_ver != hello.manifest.protocol_ver {
2463 return Ok(vec![control_error_frame(
2464 &frame,
2465 "invalid_manifest",
2466 format!(
2467 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2468 hello.protocol_ver, hello.manifest.protocol_ver
2469 ),
2470 )?]);
2471 }
2472
2473 if hello.manifest.module_id.trim().is_empty() {
2474 return Ok(vec![control_error_frame(
2475 &frame,
2476 "invalid_manifest",
2477 "manifest module_id must not be empty",
2478 )?]);
2479 }
2480
2481 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2482 Ok(negotiated_ver) => negotiated_ver,
2483 Err(message) => {
2484 return Ok(vec![control_error_frame(
2485 &frame,
2486 "version_unsupported",
2487 message,
2488 )?])
2489 }
2490 };
2491
2492 let swap_admission = self
2500 .supervisor
2501 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2502 if swap_admission == SwapHelloAdmission::Refused {
2503 warn!(
2504 module_id = %hello.manifest.module_id,
2505 connection_id = connection_id.get(),
2506 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2507 );
2508 return Ok(vec![control_error_frame(
2509 &frame,
2510 "swap_token_invalid",
2511 format!(
2512 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2513 hello.manifest.module_id
2514 ),
2515 )?]);
2516 }
2517 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2518
2519 let nonce_verified = if swap_candidate {
2524 true
2525 } else {
2526 match self.supervisor.reserved_hello_verification(
2527 &hello.manifest.module_id,
2528 hello.launch_nonce.as_deref(),
2529 ) {
2530 Ok(verified) => verified,
2531 Err(rejection) => {
2532 let message = match rejection {
2533 ReservedHelloRejection::Exact { module_id } => format!(
2534 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2535 ),
2536 ReservedHelloRejection::Prefix {
2537 prefix,
2538 owner_module_id,
2539 } => format!(
2540 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2541 hello.manifest.module_id
2542 ),
2543 };
2544 return Ok(vec![control_error_frame(
2545 &frame,
2546 "reserved_module",
2547 message,
2548 )?]);
2549 }
2550 }
2551 };
2552
2553 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2554 &hello.manifest.module_id,
2555 hello.manifest.capabilities.as_ref(),
2556 );
2557 if let Some(refusal) = reserved_capability_refusals.first() {
2558 let capability = refusal.capability.clone();
2559 let bound_module = refusal.claimants[0].clone();
2560 log_duplicate_claim_events(reserved_capability_refusals);
2561 return Ok(vec![control_error_frame(
2562 &frame,
2563 "reserved_capability",
2564 format!(
2565 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2566 capability, bound_module, hello.manifest.module_id
2567 ),
2568 )?]);
2569 }
2570
2571 if self
2574 .forwarding
2575 .connection_has_client_routes(connection_id)
2576 .map_err(RouterError::Forwarding)?
2577 {
2578 return Ok(vec![control_error_frame(
2579 &frame,
2580 "invalid_hello",
2581 "connection has open client routes and cannot also register as a module",
2582 )?]);
2583 }
2584
2585 self.hello_launch_nonces
2590 .lock()
2591 .unwrap_or_else(|poisoned| poisoned.into_inner())
2592 .record(connection_id, hello.launch_nonce.as_deref());
2593 let control_ops = effective_module_control_ops(hello.control_ops);
2594 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2597 if swap_candidate {
2598 return self.register_swap_candidate(
2599 connection_id,
2600 sink,
2601 &frame,
2602 hello.manifest,
2603 negotiated_ver,
2604 control_ops,
2605 hello_ack,
2606 );
2607 }
2608 let registration = match self
2615 .registry
2616 .register_unpublished_verified_with_control_ops(
2617 hello.manifest,
2618 negotiated_ver,
2619 connection_id,
2620 control_ops,
2621 nonce_verified,
2622 ) {
2623 Ok(registration) => registration,
2624 Err(RegistryError::DuplicateModuleId { module_id }) => {
2625 return Ok(vec![control_error_frame(
2626 &frame,
2627 "duplicate_module_id",
2628 format!(
2629 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2630 ),
2631 )?])
2632 }
2633 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2634 return Ok(vec![control_error_frame(
2635 &frame,
2636 "invalid_module_id",
2637 err.to_string(),
2638 )?])
2639 }
2640 Err(err) => {
2641 return Ok(vec![control_error_frame(
2642 &frame,
2643 "registry_error",
2644 err.to_string(),
2645 )?])
2646 }
2647 };
2648 #[cfg(test)]
2649 self.hold_between_registry_and_forwarding();
2650
2651 let reply = if let Some(sink) = sink {
2652 let concurrency = manifest_concurrency(®istration.manifest);
2668 if let Err(err) = self.forwarding.register_module_connection_acked(
2669 connection_id,
2670 registration.manifest.module_id.clone(),
2671 negotiated_ver,
2672 concurrency,
2673 sink,
2674 hello_ack,
2675 ) {
2676 if matches!(
2681 self.deregister_connection(
2682 connection_id,
2683 RegistrationEndReason::RegistrationFailed,
2684 ),
2685 Ok(r) if !r.is_empty()
2686 ) {
2687 crate::supervise::notify_registration_release();
2688 }
2689 return Ok(vec![control_error_frame(
2690 &frame,
2691 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2692 "invalid_hello"
2693 } else {
2694 forwarding_error_code(&err)
2695 },
2696 err.to_string(),
2697 )?]);
2698 }
2699 Vec::new()
2700 } else {
2701 vec![hello_ack]
2704 };
2705
2706 if !self
2711 .registry
2712 .publish_registration(connection_id)
2713 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2714 {
2715 debug!(
2716 module_id = %registration.manifest.module_id,
2717 connection_id = connection_id.get(),
2718 "registration was removed before it could be published"
2719 );
2720 }
2721
2722 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2731 info!(
2732 module_id = %registration.manifest.module_id,
2733 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2734 );
2735 }
2736
2737 self.apply_registration_capabilities(®istration);
2738
2739 self.supervisor
2740 .record_activity_registration(®istration.manifest.module_id);
2741
2742 info!(
2743 module_id = %registration.manifest.module_id,
2744 module_version = %registration.manifest.module_version,
2745 negotiated_ver,
2746 routable_provider = manifest_provides_routable_role(®istration.manifest),
2747 connection_id = connection_id.get(),
2748 "module registered"
2749 );
2750
2751 Ok(reply)
2752 }
2753
2754 #[allow(clippy::too_many_arguments)]
2771 fn register_swap_candidate(
2772 &self,
2773 connection_id: ConnectionId,
2774 sink: Option<crate::FrameSink>,
2775 frame: &Frame,
2776 manifest: ModuleManifest,
2777 negotiated_ver: u8,
2778 control_ops: Vec<String>,
2779 hello_ack: Frame,
2780 ) -> Result<Vec<Frame>, RouterError> {
2781 let module_id = manifest.module_id.clone();
2782 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2783 manifest,
2784 negotiated_ver,
2785 connection_id,
2786 control_ops,
2787 ) {
2788 Ok(registration) => registration,
2789 Err(RegistryError::DuplicateModuleId { module_id }) => {
2790 return Ok(vec![control_error_frame(
2791 frame,
2792 "duplicate_module_id",
2793 format!(
2794 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
2795 ),
2796 )?])
2797 }
2798 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2799 return Ok(vec![control_error_frame(
2800 frame,
2801 "invalid_module_id",
2802 err.to_string(),
2803 )?])
2804 }
2805 Err(err) => {
2806 return Ok(vec![control_error_frame(
2807 frame,
2808 "registry_error",
2809 err.to_string(),
2810 )?])
2811 }
2812 };
2813 #[cfg(test)]
2814 self.hold_between_registry_and_forwarding();
2815 let reply = if let Some(sink) = sink {
2816 let concurrency = manifest_concurrency(®istration.manifest);
2820 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
2821 connection_id,
2822 module_id.clone(),
2823 negotiated_ver,
2824 concurrency,
2825 sink,
2826 hello_ack,
2827 ) {
2828 if matches!(
2829 self.deregister_connection(
2830 connection_id,
2831 RegistrationEndReason::RegistrationFailed,
2832 ),
2833 Ok(r) if !r.is_empty()
2834 ) {
2835 crate::supervise::notify_registration_release();
2836 }
2837 return Ok(vec![control_error_frame(
2838 frame,
2839 forwarding_error_code(&err),
2840 err.to_string(),
2841 )?]);
2842 }
2843 Vec::new()
2844 } else {
2845 vec![hello_ack]
2846 };
2847 if !self
2848 .registry
2849 .publish_registration(connection_id)
2850 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2851 {
2852 debug!(
2853 module_id = %module_id,
2854 connection_id = connection_id.get(),
2855 "swap candidate registration was removed before it could be published"
2856 );
2857 }
2858 self.supervisor.mark_swap_candidate_admitted(&module_id);
2859 info!(
2860 module_id = %module_id,
2861 module_version = %registration.manifest.module_version,
2862 negotiated_ver,
2863 ready = registration.ready,
2864 connection_id = connection_id.get(),
2865 "swap candidate registered; not routable until cutover"
2866 );
2867 Ok(reply)
2868 }
2869
2870 fn build_hello_ack(
2871 &self,
2872 frame: &Frame,
2873 negotiated_ver: u8,
2874 module_id: &str,
2875 ) -> Result<Frame, RouterError> {
2876 let ack = ModuleHelloAckBody {
2877 negotiated_ver,
2878 subc_ops: module_subc_ops(),
2879 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
2880 storage: self
2881 .storage_config
2882 .as_ref()
2883 .map(|cfg| cfg.descriptor_for(module_id)),
2884 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
2885 };
2886 let body = serde_json::to_vec(&ack).map_err(|err| {
2887 RouterError::backend(
2888 0,
2889 frame.header.corr,
2890 format!("failed to encode HELLO_ACK: {err}"),
2891 )
2892 })?;
2893
2894 Frame::build_with_version(
2895 negotiated_ver,
2896 FrameType::HelloAck,
2897 control_flags(),
2898 0,
2899 0,
2900 frame.header.corr,
2901 body,
2902 )
2903 .map_err(RouterError::FrameBuild)
2904 }
2905
2906 async fn handle_client_control_request(
2907 &self,
2908 ctx: &RouteCtx,
2909 frame: Frame,
2910 request: ClientControlRequest,
2911 route_open_timings: &mut Option<RouteOpenTimings>,
2912 ) -> Result<Vec<Frame>, RouterError> {
2913 if let Some(operation) = privileged_operation(&request) {
2914 let caller = self.resolve_caller(ctx.connection_id).await;
2915 let op = client_control_request_op(&request);
2916 let text = prompt_text(operation, &caller);
2917 match self.privileged_operator_prompt {
2918 PrivilegedOperatorPrompt::Log => {
2919 info!(target: "subc_daemon::privileged_prompt", operation = op,
2920 target = ?operation, caller = %caller.description(), summary = %text,
2921 "privileged request would prompt the operator");
2922 }
2923 PrivilegedOperatorPrompt::Enforce => {
2924 let key =
2925 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
2926 let outcome = self
2927 .forwarding
2928 .operator_confirms()
2929 .approve_privileged(ctx, &frame, op, text, caller.description(), key)
2930 .await;
2931 if outcome.code != "confirmed" {
2932 return Ok(vec![outcome.frame(&frame)]);
2933 }
2934 }
2935 }
2936 }
2937 match request {
2938 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
2939 ClientControlRequest::CatalogList { module_id } => {
2940 self.handle_catalog_list(frame, module_id)
2941 }
2942 ClientControlRequest::RouteOpen {
2943 target,
2944 identity,
2945 consumer_identity,
2946 consumer_capabilities,
2947 role_versions,
2948 admission_facts,
2949 scope,
2950 } => {
2951 self.handle_route_open(
2952 ctx,
2953 frame,
2954 RouteOpenRequest {
2955 target,
2956 identity,
2957 consumer_identity,
2958 consumer_capabilities,
2959 role_versions,
2960 admission_facts,
2961 scope,
2962 },
2963 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
2964 )
2965 .await
2966 }
2967 ClientControlRequest::RoutePoll {
2968 route_channel,
2969 route_epoch,
2970 kind,
2971 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
2972 ClientControlRequest::SupervisorList {} => self.handle_supervisor_list(frame).await,
2973 ClientControlRequest::SupervisorSpawnSnapshot {} => {
2974 self.handle_supervisor_spawn_snapshot(frame)
2975 }
2976 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
2977 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
2978 }
2979 ClientControlRequest::SupervisorRestart {
2980 module_id,
2981 drain_timeout_ms,
2982 } => {
2983 self.log_supervisor_request_received(
2984 ctx,
2985 frame.header.corr,
2986 ops::SUPERVISOR_RESTART,
2987 Some(&module_id),
2988 None,
2989 )?;
2990 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
2991 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
2992 .await
2993 }
2994 ClientControlRequest::SupervisorSwap {
2995 module_id,
2996 ready_timeout_ms,
2997 } => {
2998 self.log_supervisor_request_received(
2999 ctx,
3000 frame.header.corr,
3001 ops::SUPERVISOR_SWAP,
3002 Some(&module_id),
3003 None,
3004 )?;
3005 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3006 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3007 .await
3008 }
3009 ClientControlRequest::SupervisorReload { module_id } => {
3010 self.log_supervisor_request_received(
3011 ctx,
3012 frame.header.corr,
3013 ops::SUPERVISOR_RELOAD,
3014 Some(&module_id),
3015 None,
3016 )?;
3017 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3018 self.handle_supervisor_reload(frame, module_id, requester)
3019 .await
3020 }
3021 ClientControlRequest::SupervisorRescan { preview } => {
3022 if !preview {
3023 self.log_supervisor_request_received(
3024 ctx,
3025 frame.header.corr,
3026 ops::SUPERVISOR_RESCAN,
3027 None,
3028 None,
3029 )?;
3030 }
3031 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3032 self.handle_supervisor_rescan(
3033 frame,
3034 if preview {
3035 RescanMode::Preview
3036 } else {
3037 RescanMode::Execute
3038 },
3039 requester,
3040 )
3041 .await
3042 }
3043 ClientControlRequest::SupervisorRescanChecked {
3044 preview,
3045 candidate,
3046 expect,
3047 } => {
3048 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect)
3049 .await
3050 }
3051 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3052 self.log_supervisor_request_received(
3053 ctx,
3054 frame.header.corr,
3055 ops::SUPERVISOR_RELEASE_RESERVED,
3056 Some(&module_id),
3057 None,
3058 )?;
3059 self.handle_supervisor_release_reserved(frame, module_id)
3060 .await
3061 }
3062 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3063 self.log_supervisor_request_received(
3064 ctx,
3065 frame.header.corr,
3066 ops::SUPERVISOR_SET_ENABLED,
3067 Some(&module_id),
3068 Some(enabled),
3069 )?;
3070 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3071 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3072 .await
3073 }
3074 ClientControlRequest::SupervisorHealthProbe { module_id } => {
3075 self.handle_supervisor_health_probe(frame, module_id).await
3076 }
3077 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3078 ClientControlRequest::SupervisorRoutes { module_id } => {
3079 self.handle_supervisor_routes(frame, module_id)
3080 }
3081 ClientControlRequest::SupervisorProvenance { module_id } => {
3082 self.handle_supervisor_provenance(frame, module_id).await
3083 }
3084 ClientControlRequest::SupervisorStderrTail {
3085 module_id,
3086 max_lines,
3087 max_bytes,
3088 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3089 ClientControlRequest::SupervisorTerminals { module_id } => {
3090 self.handle_supervisor_terminals(frame, module_id).await
3091 }
3092 }
3093 }
3094
3095 fn log_supervisor_request_received(
3096 &self,
3097 ctx: &RouteCtx,
3098 corr: u64,
3099 op: &'static str,
3100 module_id: Option<&str>,
3101 enabled: Option<bool>,
3102 ) -> Result<(), RouterError> {
3103 let caller = self.supervisor_requester(ctx, corr)?;
3104 let caller = principal_label(&caller);
3105 info!(
3108 target: "control",
3109 op,
3110 module_id,
3111 enabled,
3112 connection_id = ctx.connection_id.get(),
3113 caller = %caller,
3114 "supervisor request received"
3115 );
3116 Ok(())
3117 }
3118
3119 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3120 Ok(self
3121 .registry
3122 .get_module_by_connection(ctx.connection_id)
3123 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3124 .map(|registration| Principal::Reserved {
3125 module_id: registration.manifest.module_id,
3126 })
3127 .unwrap_or(Principal::Direct))
3128 }
3129
3130 fn handle_module_control_request(
3131 &self,
3132 connection_id: ConnectionId,
3133 frame: Frame,
3134 request: ModuleControlRequestFromModule,
3135 ) -> Result<Vec<Frame>, RouterError> {
3136 match request {
3137 ModuleControlRequestFromModule::CatalogUpdate {
3138 provides,
3139 capabilities,
3140 ready,
3141 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3142 ModuleControlRequestFromModule::LiveRoots {} => {
3143 let registered = self
3144 .registry
3145 .get_module_by_connection(connection_id)
3146 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3147 let Some(registration) = registered else {
3148 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3149 };
3150 let response = self
3151 .forwarding
3152 .live_roots(®istration.manifest.module_id)
3153 .map_err(RouterError::Forwarding)?;
3154 Ok(vec![control_response_body_frame(
3155 &frame,
3156 &response,
3157 "ModuleControlResponseToModule::LiveRoots",
3158 )?])
3159 }
3160 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3161 self.handle_scope_sync(connection_id, frame, generation, scopes)
3162 }
3163 ModuleControlRequestFromModule::ScopeApply {
3164 generation,
3165 upsert,
3166 end,
3167 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3168 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3169 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3170 }
3171 }
3172 }
3173
3174 fn handle_scope_served_by(
3175 &self,
3176 connection_id: ConnectionId,
3177 frame: Frame,
3178 ) -> Result<Vec<Frame>, RouterError> {
3179 let registered = self
3180 .registry
3181 .get_module_by_connection(connection_id)
3182 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3183 let Some(registered) = registered else {
3184 return Ok(vec![control_error_frame(
3185 &frame,
3186 "not_registered",
3187 "scope.served_by requires an active module registration owned by this connection",
3188 )?]);
3189 };
3190 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3191 {
3192 Ok(request) => request,
3193 Err(err) => {
3194 return Ok(vec![control_error_frame(
3195 &frame,
3196 "invalid_control_body",
3197 format!("malformed scope.served_by body: {err}"),
3198 )?])
3199 }
3200 };
3201 let answer = self
3202 .scopes
3203 .read()
3204 .unwrap_or_else(|poisoned| poisoned.into_inner())
3205 .served_by(
3206 ®istered.manifest.module_id,
3207 &request.scope_ref,
3208 request.scope_epoch,
3209 &request.module_id,
3210 );
3211 let reply = subc_control::ScopeServedByReply::new(
3212 answer,
3213 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3214 );
3215 Ok(vec![control_response_body_frame(
3216 &frame,
3217 &reply,
3218 "ScopeServedByReply",
3219 )?])
3220 }
3221
3222 fn handle_operator_confirm(
3223 &self,
3224 ctx: &RouteCtx,
3225 frame: Frame,
3226 ) -> Result<Vec<Frame>, RouterError> {
3227 use crate::operator_confirm::{audit, Outcome};
3228 let registration = self
3229 .registry
3230 .get_module_by_connection(ctx.connection_id)
3231 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3232 let Some(registration) = registration else {
3233 let outcome = Outcome::refusal("not_registered");
3234 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3235 return Ok(vec![outcome.frame(&frame)]);
3236 };
3237 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3238 Ok(request) => request,
3239 Err(_) => {
3240 let outcome = Outcome::refusal("invalid_control_body");
3241 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3242 return Ok(vec![outcome.frame(&frame)]);
3243 }
3244 };
3245 let module_id = registration.manifest.module_id;
3246 let nonce = self
3250 .hello_launch_nonces
3251 .lock()
3252 .unwrap_or_else(|p| p.into_inner())
3253 .nonce(ctx.connection_id)
3254 .map(str::to_owned);
3255 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3256 self.supervisor
3257 .spawned_consumer_authorized(&module_id, nonce)
3258 });
3259 let confirms = self.forwarding.operator_confirms();
3260 self.forwarding
3261 .with_operator_route(
3262 ctx.connection_id,
3263 request.route_channel,
3264 request.route_epoch,
3265 |binding| confirms.admit(ctx, frame, module_id, nonce_proven, request, binding),
3266 )
3267 .map_err(RouterError::Forwarding)
3268 }
3269
3270 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3273 let mut scopes = ScopeRecords::from(scopes);
3274 for (index, record) in scopes.records.iter_mut().enumerate() {
3275 if let Some(refusal) = Self::canonicalize_workdir(record) {
3276 scopes.refusals.insert(index, refusal);
3277 }
3278 }
3279 scopes
3280 }
3281
3282 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3283 let workdir = record.workdir.as_mut()?;
3284 let path = std::path::Path::new(workdir);
3285 if !path.is_absolute() {
3286 return Some((
3287 error_codes::INVALID_WORKDIR,
3288 "workdir must be absolute".to_string(),
3289 ));
3290 }
3291 match ProjectRootId::from_path_allowing_missing(path) {
3292 Ok(root) => match root.as_path().to_str() {
3293 Some(canonical) => {
3294 *workdir = canonical.to_string();
3295 None
3296 }
3297 None => Some((
3298 error_codes::INVALID_WORKDIR,
3299 "canonical workdir is not UTF-8".to_string(),
3300 )),
3301 },
3302 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3303 }
3304 }
3305
3306 fn handle_scope_sync(
3310 &self,
3311 connection_id: ConnectionId,
3312 frame: Frame,
3313 generation: u64,
3314 scopes: Vec<ScopeRecord>,
3315 ) -> Result<Vec<Frame>, RouterError> {
3316 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3317 }
3318
3319 fn handle_scope_change(
3320 &self,
3321 connection_id: ConnectionId,
3322 frame: Frame,
3323 generation: u64,
3324 scopes: Vec<ScopeRecord>,
3325 end: Option<Vec<ScopeEnd>>,
3326 ) -> Result<Vec<Frame>, RouterError> {
3327 let is_apply = end.is_some();
3328 let Some(registration) = self
3329 .registry
3330 .get_module_by_connection(connection_id)
3331 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3332 else {
3333 return Ok(vec![control_error_frame(
3334 &frame,
3335 "not_registered",
3336 "scope.sync and scope.apply require an active module registration owned by this connection",
3337 )?]);
3338 };
3339 let owner = registration.manifest.module_id;
3340 let scopes = Self::prepare_scope_records(scopes);
3341 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3342 let is_current_launch = |connection: ConnectionId| {
3343 self.hello_launch_nonces
3344 .lock()
3345 .unwrap_or_else(|poisoned| poisoned.into_inner())
3346 .presented(connection, current_nonce.as_deref())
3347 };
3348 let mut table = self
3353 .scopes
3354 .write()
3355 .unwrap_or_else(|poisoned| poisoned.into_inner());
3356 let outcome = match end {
3357 Some(end) => table.apply(
3358 &owner,
3359 connection_id,
3360 is_current_launch,
3361 generation,
3362 scopes,
3363 end,
3364 ),
3365 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3366 };
3367 let drained = match &outcome {
3368 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3369 Err(_) => Vec::new(),
3370 };
3371 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3372 drop(table);
3373 if has_deadlines {
3374 self.scope_deadline_added.notify_one();
3375 }
3376 match outcome {
3377 Ok(applied) => {
3378 let counts = ScopeOutcomeCounts::of(&applied.results);
3379 info!(
3380 owner = %owner,
3381 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3382 generation,
3383 records = applied.results.len(),
3384 created = counts.created,
3385 replaced = counts.replaced,
3386 updated = counts.updated,
3387 unchanged = counts.unchanged,
3388 refused = counts.refused,
3389 ended = applied.ended.len(),
3390 tag_changes = applied.tag_changes.len(),
3391 routes_closed = drained.len(),
3392 "scope change accepted"
3393 );
3394 for refused in applied
3400 .results
3401 .iter()
3402 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3403 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3404 {
3405 warn!(
3406 owner = %owner,
3407 generation,
3408 scope_ref = %refused.scope_ref,
3409 scope_epoch = refused.scope_epoch,
3410 code = refused.code.as_deref().unwrap_or(""),
3411 "scope record refused"
3412 );
3413 }
3414 self.close_scope_drained_routes(drained);
3415 let response = if is_apply {
3416 ModuleControlResponseToModule::ScopeApply {
3417 generation,
3418 results: applied.results,
3419 end_results: applied.end_results,
3420 ended: applied.ended,
3421 }
3422 } else {
3423 ModuleControlResponseToModule::ScopeSync {
3424 generation,
3425 results: applied.results,
3426 ended: applied.ended,
3427 }
3428 };
3429 Ok(vec![control_response_body_frame(
3430 &frame,
3431 &response,
3432 "scope change response",
3433 )?])
3434 }
3435 Err(refusal) => {
3436 info!(
3437 owner = %owner,
3438 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3439 generation,
3440 code = refusal.code,
3441 "scope change refused"
3442 );
3443 Ok(vec![control_error_frame(
3444 &frame,
3445 refusal.code,
3446 refusal.message,
3447 )?])
3448 }
3449 }
3450 }
3451
3452 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3458 if drained.is_empty() {
3459 return;
3460 }
3461 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3462 BTreeMap::new();
3463 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3464 for route in drained {
3465 warn!(
3466 module_id = %route.module_id,
3467 reason = ?route.reason,
3468 client_connection_id = route.client.connection_id.get(),
3469 route_channel = route.client.channel,
3470 "closing route because its scope changed"
3471 );
3472 pushes
3473 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3474 .or_insert_with(|| (route.reason, Vec::new()))
3475 .1
3476 .push(EndpointRoute {
3477 goodbye_target: route.client.clone(),
3478 principal: Principal::Unverified,
3479 bound_at: Instant::now(),
3480 draining: false,
3481 drain_reason: None,
3482 });
3483 goodbyes.push(route.module);
3484 goodbyes.push(route.client);
3485 }
3486 for ((module_id, _), (reason, routes)) in pushes {
3487 send_route_control_pushes(
3488 &self.forwarding,
3489 routes,
3490 ClientControlPush::RouteClosed {
3491 module_id,
3492 channels: Vec::new(),
3493 reason,
3494 drained: false,
3495 abandoned: 0,
3496 excluded_subscriptions: 0,
3497 terminal: Some(false),
3498 },
3499 );
3500 }
3501 self.emit_route_goodbyes(goodbyes);
3502 }
3503
3504 fn handle_scope_describe(
3507 &self,
3508 connection_id: ConnectionId,
3509 frame: Frame,
3510 owner: Principal,
3511 scope_ref: String,
3512 ) -> Result<Vec<Frame>, RouterError> {
3513 let registered = self
3514 .registry
3515 .get_module_by_connection(connection_id)
3516 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3517 let Some(registered) = registered else {
3518 return Ok(vec![control_error_frame(
3519 &frame,
3520 "not_registered",
3521 "scope.describe requires an active module registration owned by this connection",
3522 )?]);
3523 };
3524 let mut description = self
3525 .scopes
3526 .read()
3527 .unwrap_or_else(|poisoned| poisoned.into_inner())
3528 .describe(&owner, &scope_ref);
3529 if !registered
3532 .manifest
3533 .capabilities
3534 .as_ref()
3535 .is_some_and(|caps| {
3536 caps.provides
3537 .iter()
3538 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3539 })
3540 {
3541 if let Some(stamp) = description.stamp.as_mut() {
3542 stamp.attributes.on_behalf_of_agent = None;
3543 }
3544 }
3545 if !registered
3546 .manifest
3547 .capabilities
3548 .as_ref()
3549 .is_some_and(|caps| {
3550 caps.provides
3551 .iter()
3552 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3553 })
3554 {
3555 if let Some(stamp) = description.stamp.as_mut() {
3556 stamp.attributes.flow_version = None;
3557 stamp.attributes.flow_code_hash = None;
3558 stamp.attributes.flow_install_id = None;
3559 }
3560 }
3561 let owner_configured = match &owner {
3562 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3567 _ => false,
3568 };
3569 let response = ModuleControlResponseToModule::ScopeDescribe {
3570 status: description.status,
3571 scope_epoch: description.scope_epoch,
3572 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3573 owner_synced: description.owner_synced,
3574 owner_configured,
3575 scope: description.stamp,
3576 };
3577 Ok(vec![control_response_body_frame(
3578 &frame,
3579 &response,
3580 "ModuleControlResponseToModule::ScopeDescribe",
3581 )?])
3582 }
3583
3584 fn handle_catalog_update(
3585 &self,
3586 connection_id: ConnectionId,
3587 frame: Frame,
3588 provides: Vec<ProviderRole>,
3589 capabilities: Option<CapabilityDeclarations>,
3590 ready: Option<bool>,
3591 ) -> Result<Vec<Frame>, RouterError> {
3592 self.refresh_capability_requirements();
3593 let Some(registration) = self
3594 .registry
3595 .get_module_by_connection(connection_id)
3596 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3597 else {
3598 return Ok(vec![control_error_frame(
3599 &frame,
3600 "not_registered",
3601 "catalog.update requires an active module registration owned by this connection",
3602 )?]);
3603 };
3604
3605 if let Some(message) =
3606 catalog_update_frozen_field_message(®istration.manifest, &provides)
3607 {
3608 return Ok(vec![control_error_frame(
3609 &frame,
3610 "catalog_update_frozen_field",
3611 message,
3612 )?]);
3613 }
3614
3615 let mut candidate = registration.manifest.clone();
3616 candidate.provides = provides.clone();
3617 candidate.capabilities = capabilities
3618 .clone()
3619 .or_else(|| registration.manifest.capabilities.clone());
3620 if let Some(rich_ui) = &candidate.rich_ui {
3621 if let Err(err) = rich_ui.validate() {
3622 return Ok(vec![control_error_frame(
3623 &frame,
3624 "invalid_manifest",
3625 err.to_string(),
3626 )?]);
3627 }
3628 }
3629 if let Err(err) = candidate.validate_capability_grammar() {
3630 return Ok(vec![control_error_frame(
3631 &frame,
3632 "invalid_capability_grammar",
3633 err.to_string(),
3634 )?]);
3635 }
3636
3637 let mut conflicts = self
3640 .capability_evaluator
3641 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
3642 if let Some(conflict) = conflicts.first() {
3643 let message = format!(
3644 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
3645 conflict.capability, conflict.claimants[0], candidate.module_id
3646 );
3647 for conflict in &mut conflicts {
3648 conflict.source = DuplicateClaimSource::CatalogUpdate;
3649 }
3650 log_duplicate_claim_events(conflicts);
3651 return Ok(vec![control_error_frame(
3652 &frame,
3653 "reserved_capability",
3654 message,
3655 )?]);
3656 }
3657
3658 let updated = self
3659 .registry
3660 .replace_catalog_for_connection(connection_id, provides, capabilities, ready)
3661 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3662 if updated.is_none() {
3663 return Ok(vec![control_error_frame(
3664 &frame,
3665 "not_registered",
3666 "catalog.update requires an active module registration owned by this connection",
3667 )?]);
3668 }
3669 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
3670 log_duplicate_claim_events(
3671 self.capability_evaluator
3672 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
3673 );
3674 }
3675 if capability_census_trigger(
3676 registration.manifest.capabilities.as_ref(),
3677 updated
3678 .as_ref()
3679 .and_then(|entry| entry.manifest.capabilities.as_ref()),
3680 ) {
3681 self.enforce_capability_denies();
3682 }
3683 self.refresh_capability_requirements();
3684
3685 let response = ModuleControlResponseToModule::CatalogUpdate {};
3686 control_response_body_frame(
3687 &frame,
3688 &response,
3689 "ModuleControlResponseToModule::CatalogUpdate",
3690 )
3691 .map(|frame| vec![frame])
3692 }
3693
3694 pub(crate) fn with_activity_ledger_config(
3695 mut self,
3696 run_dir: &Path,
3697 incarnation: String,
3698 config: crate::activity::ActivityConfig,
3699 ) -> Self {
3700 self.activity = Some(crate::activity::ActivityLedger::open_config(
3701 run_dir.join("activity.jsonl"),
3702 incarnation,
3703 config,
3704 ));
3705 self.supervisor.set_activity(self.activity.clone());
3706 self
3707 }
3708
3709 pub(crate) async fn configure_activity(
3710 &self,
3711 config: crate::activity::ActivityConfig,
3712 ) -> Result<(), &'static str> {
3713 match &self.activity {
3714 Some(ledger) => ledger.configure(config).await,
3715 None => Ok(()),
3716 }
3717 }
3718
3719 fn cancel_activity_subscription(
3720 &self,
3721 ctx: &RouteCtx,
3722 frame: &Frame,
3723 ) -> Result<bool, RouterError> {
3724 let task = self
3725 .activity_streams
3726 .lock()
3727 .unwrap_or_else(|p| p.into_inner())
3728 .remove(&(ctx.connection_id, frame.header.corr));
3729 let Some(task) = task else {
3730 return Ok(false);
3731 };
3732 task.abort();
3733 let end = Frame::build_with_version(
3734 response_version(frame),
3735 FrameType::StreamEnd,
3736 control_flags(),
3737 0,
3738 0,
3739 frame.header.corr,
3740 Vec::new(),
3741 )
3742 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
3743 let sink = ctx.egress.clone();
3744 tokio::spawn(async move {
3745 let _ = sink.send(end).await;
3746 });
3747 Ok(true)
3748 }
3749
3750 async fn handle_activity_read(
3751 &self,
3752 ctx: &RouteCtx,
3753 frame: Frame,
3754 ) -> Result<Vec<Frame>, RouterError> {
3755 use subc_control::{
3756 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
3757 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
3758 };
3759 #[derive(Deserialize)]
3760 struct Identity {
3761 #[serde(default)]
3762 consumer_identity: Option<ConsumerIdentity>,
3763 }
3764 let principal = match self
3768 .forwarding
3769 .module_id_for_connection(ctx.connection_id)
3770 .map_err(RouterError::Forwarding)?
3771 {
3772 Some(module_id) => Ok(Principal::Reserved { module_id }),
3773 None => {
3774 let identity: Identity = match serde_json::from_slice(&frame.body) {
3775 Ok(identity) => identity,
3776 Err(_) => {
3777 return Ok(vec![control_error_frame(
3778 &frame,
3779 ACTIVITY_MALFORMED,
3780 "malformed activity read",
3781 )?])
3782 }
3783 };
3784 self.route_open_principal(&frame, identity.consumer_identity)?
3785 }
3786 };
3787 let admitted = matches!(principal, Ok(Principal::Direct))
3788 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
3789 if !admitted {
3790 return Ok(vec![control_error_frame(
3791 &frame,
3792 ACTIVITY_READ_FORBIDDEN,
3793 "activity reads require an operator or trusted mirror principal",
3794 )?]);
3795 }
3796 let Some(ledger) = &self.activity else {
3797 return Ok(vec![control_error_frame(
3798 &frame,
3799 ACTIVITY_UNAVAILABLE,
3800 "activity ledger unavailable",
3801 )?]);
3802 };
3803 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
3804 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
3805 {
3806 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
3807 Ok(request) => ledger.query(request).await,
3808 Err(_) => Err(ACTIVITY_MALFORMED),
3809 };
3810 return match result {
3811 Ok(reply) => Ok(vec![control_response_body_frame(
3812 &frame,
3813 &reply,
3814 ops::ACTIVITY_QUERY,
3815 )?]),
3816 Err(code) => Ok(vec![control_error_frame(
3817 &frame,
3818 code,
3819 "activity query refused",
3820 )?]),
3821 };
3822 }
3823 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
3824 Ok(request) => request,
3825 Err(_) => {
3826 return Ok(vec![control_error_frame(
3827 &frame,
3828 ACTIVITY_MALFORMED,
3829 "malformed activity subscription",
3830 )?])
3831 }
3832 };
3833 let mut subscription = match ledger.subscribe(request.since).await {
3834 Ok(subscription) => subscription,
3835 Err(code) => {
3836 return Ok(vec![control_error_frame(
3837 &frame,
3838 code,
3839 "activity subscription refused",
3840 )?])
3841 }
3842 };
3843 let key = (ctx.connection_id, frame.header.corr);
3844 let sink = ctx.egress.clone();
3845 let mut streams = self
3846 .activity_streams
3847 .lock()
3848 .unwrap_or_else(|p| p.into_inner());
3849 streams.retain(|_, task| !task.is_finished());
3850 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
3851 return Ok(vec![control_error_frame(
3852 &frame,
3853 "duplicate_subscription",
3854 "activity subscription correlation id already in use",
3855 )?]);
3856 }
3857 let task = tokio::spawn(async move {
3858 loop {
3859 let entry = if let Some(entry) = subscription.replay.pop_front() {
3860 Some(entry)
3861 } else {
3862 subscription.receiver.recv().await
3863 };
3864 let Some(entry) = entry else {
3865 break;
3866 };
3867 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
3868 return;
3869 };
3870 let Ok(next) = Frame::build_with_version(
3871 response_version(&frame),
3872 FrameType::StreamData,
3873 control_flags(),
3874 0,
3875 0,
3876 frame.header.corr,
3877 body,
3878 ) else {
3879 return;
3880 };
3881 if sink.send(next).await.is_err() {
3882 return;
3883 }
3884 }
3885 if let Some(code) = subscription.terminal_code() {
3888 if let Ok(end) =
3889 control_error_frame(&frame, code, "activity subscriber fell behind")
3890 {
3891 let _ = sink.send(end).await;
3892 }
3893 }
3894 });
3895 streams.insert(key, task.abort_handle());
3896 Ok(Vec::new())
3897 }
3898
3899 async fn handle_activity_record(
3900 &self,
3901 ctx: &RouteCtx,
3902 frame: Frame,
3903 ) -> Result<Vec<Frame>, RouterError> {
3904 use subc_control::{
3905 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
3906 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
3907 };
3908 let result = async {
3909 let module = self
3910 .forwarding
3911 .module_id_for_connection(ctx.connection_id)
3912 .ok()
3913 .flatten()
3914 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
3915 let request: ActivityRecordRequest =
3916 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
3917 crate::activity::validate(&module, &request)?;
3918 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
3919 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
3920 let at_ms = std::time::SystemTime::now()
3921 .duration_since(std::time::UNIX_EPOCH)
3922 .unwrap_or_default()
3923 .as_millis() as u64;
3924 ledger.record(module, request, actor, at_ms).await
3925 }
3926 .await;
3927 match result {
3928 Ok(entry) => Ok(vec![control_response_body_frame(
3929 &frame,
3930 &ActivityRecordResponse { seq: entry.seq },
3931 "activity.record",
3932 )?]),
3933 Err(code) => Ok(vec![control_error_frame(
3934 &frame,
3935 code,
3936 "activity recording refused",
3937 )?]),
3938 }
3939 }
3940
3941 fn activity_actor(
3942 &self,
3943 connection: ConnectionId,
3944 module: &str,
3945 on: Option<&subc_control::ActivityOn>,
3946 ) -> Result<subc_control::ActivityActor, &'static str> {
3947 use crate::forwarding::{DataRoute, DataRouteState};
3948 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
3949 let principal = |id: &str| Principal::Reserved {
3950 module_id: id.to_owned(),
3951 };
3952 let mut actor = ActivityActor {
3953 principal: principal(module),
3954 scope_owner: None,
3955 agent_id: None,
3956 run_id: None,
3957 flow_id: None,
3958 project_root: None,
3959 workdir: None,
3960 project_id: None,
3961 };
3962 let scope = match on {
3963 None => return Ok(actor),
3964 Some(ActivityOn::Scope(scope)) => {
3965 let selector = subc_protocol::scope::ScopeSelector::new(
3966 principal(module),
3967 scope.scope_ref.clone(),
3968 Some(scope.epoch),
3969 );
3970 self.scopes
3971 .read()
3972 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3973 .attribute(&principal(module), module, &selector, true)
3974 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3975 }
3976 Some(ActivityOn::Route(route)) => {
3977 let DataRoute::Module(DataRouteState::Bound(binding)) = self
3978 .forwarding
3979 .lookup_data_route(connection, route.channel, route.epoch)
3980 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
3981 else {
3982 return Err(ACTIVITY_ACTOR_UNKNOWN);
3983 };
3984 if binding.module_id != module {
3985 return Err(ACTIVITY_ACTOR_UNKNOWN);
3986 }
3987 actor.principal = binding.principal.clone();
3988 actor.project_root = binding
3989 .project_root
3990 .as_ref()
3991 .map(|root| root.as_path().to_path_buf());
3992 let Some(bound) = &binding.scope else {
3993 return Ok(actor);
3994 };
3995 let owner = principal(&bound.owner);
3996 let selector = subc_protocol::scope::ScopeSelector::new(
3997 owner.clone(),
3998 bound.scope_ref.clone(),
3999 Some(bound.tag.scope_epoch),
4000 );
4001 let admitted = self
4002 .scopes
4003 .read()
4004 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4005 .attribute(&owner, module, &selector, true)
4006 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4007 if admitted.tag != bound.tag {
4008 return Err(ACTIVITY_ACTOR_UNKNOWN);
4009 }
4010 admitted
4011 }
4012 };
4013 actor.scope_owner = Some(scope.stamp.owner);
4014 actor.agent_id = scope.stamp.attributes.agent_id;
4015 actor.run_id = scope.stamp.attributes.run_id;
4016 actor.flow_id = scope.stamp.attributes.flow_id;
4017 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4018 actor.project_id = scope.stamp.project_id;
4019 Ok(actor)
4020 }
4021
4022 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4023 self.refresh_capability_requirements();
4024 let mut counters = self.counters.snapshot();
4030 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4031 obj.insert(
4032 "activity_unparseable_lines".into(),
4033 ledger.read_errors().into(),
4034 );
4035 obj.insert(
4036 "activity_lifecycle_append_failures".into(),
4037 ledger.lifecycle_failures().into(),
4038 );
4039 obj.insert(
4040 "activity_started_digest_skips".into(),
4041 ledger.lifecycle_digest_skips().into(),
4042 );
4043 }
4044 if let (Ok((connections_with_routes, max)), Some(obj)) = (
4045 self.forwarding.client_route_concentration(),
4046 counters.as_object_mut(),
4047 ) {
4048 obj.insert(
4049 "client_connections_with_routes".into(),
4050 connections_with_routes.into(),
4051 );
4052 obj.insert("max_routes_on_one_connection".into(), max.into());
4053 }
4054 if let (Some(open_breakers), Some(obj)) = (
4060 self.route_bind_breakers.open_snapshot(),
4061 counters.as_object_mut(),
4062 ) {
4063 obj.insert("route_bind_breakers_open".into(), open_breakers);
4064 }
4065 if let (Some(reason), Some(obj)) = (
4070 self.local_endpoint_unavailable.clone(),
4071 counters.as_object_mut(),
4072 ) {
4073 obj.insert("local_endpoint_unavailable".into(), reason.into());
4074 }
4075 let response = ClientControlResponse::ServerDescribe {
4076 protocol_ver: PROTOCOL_VERSION,
4077 subc_ops: subc_ops(),
4078 capabilities: self.describe_capabilities(),
4079 connected_clients: self.connected_clients.count(),
4080 counters: Some(counters),
4081 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4082 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4083 capability_requirements: self.capability_requirement_statuses(),
4084 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4085 config_path: self
4086 .rescan
4087 .as_ref()
4088 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4089 };
4090 Ok(vec![control_response_body_frame(
4091 &frame,
4092 &response,
4093 "ClientControlResponse::ServerDescribe",
4094 )?])
4095 }
4096
4097 fn handle_catalog_list(
4098 &self,
4099 frame: Frame,
4100 module_id: Option<String>,
4101 ) -> Result<Vec<Frame>, RouterError> {
4102 let (generation, modules) = self
4103 .registry
4104 .list_modules_with_nonce_verification()
4105 .map_err(|err| {
4106 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4107 })?;
4108 let entries = modules
4109 .into_iter()
4110 .filter(|(registration, _)| {
4111 module_id
4112 .as_deref()
4113 .map(|wanted| registration.manifest.module_id == wanted)
4114 .unwrap_or(true)
4115 })
4116 .map(|(registration, nonce_verified)| {
4117 let not_ready = self.not_ready_reason(®istration);
4118 let roles = registration.manifest.provides;
4119 CatalogEntry::new(
4120 registration.manifest.module_id,
4121 roles,
4122 registration.control_ops,
4123 )
4124 .with_ready(not_ready.is_none())
4125 .with_not_ready(not_ready)
4126 .with_module_version(Some(registration.manifest.module_version))
4127 .with_capabilities(registration.manifest.capabilities)
4128 .with_self_signals(registration.manifest.self_signals)
4129 .with_events(registration.manifest.events)
4130 .with_operation_delivery(registration.manifest.operation_delivery)
4131 .with_operation_payload(registration.manifest.operation_payload)
4132 .with_flow_operations(registration.manifest.flow_operations)
4133 .with_confirm_operations(registration.manifest.confirm_operations)
4134 .with_rich_ui(registration.manifest.rich_ui)
4135 .with_role_declarations(registration.manifest.role_declarations)
4136 .with_nonce_verified(nonce_verified)
4137 })
4138 .collect();
4139 let response = ClientControlResponse::CatalogList {
4140 generation,
4141 modules: entries,
4142 subc_ops: subc_ops(),
4143 };
4144 Ok(vec![control_response_body_frame(
4145 &frame,
4146 &response,
4147 "ClientControlResponse::CatalogList",
4148 )?])
4149 }
4150
4151 fn attest_route_open_consumer(
4156 &self,
4157 consumer_identity: Option<&ConsumerIdentity>,
4158 ) -> Option<Principal> {
4159 let Some(consumer_identity) = consumer_identity else {
4160 return Some(Principal::Direct);
4161 };
4162 self.supervisor
4163 .spawned_consumer_authorized(
4164 &consumer_identity.module_id,
4165 &consumer_identity.launch_nonce,
4166 )
4167 .then(|| Principal::Reserved {
4168 module_id: consumer_identity.module_id.clone(),
4169 })
4170 }
4171
4172 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4176 let Ok(ClientControlRequest::RouteOpen {
4177 identity,
4178 consumer_identity,
4179 scope,
4180 ..
4181 }) = parse_client_control_request(&frame.body)
4182 else {
4183 return RouteOpenCaller::unknown();
4184 };
4185 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4186 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4187 }
4188
4189 fn route_open_principal(
4190 &self,
4191 frame: &Frame,
4192 consumer_identity: Option<ConsumerIdentity>,
4193 ) -> Result<Result<Principal, Frame>, RouterError> {
4194 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4195 self.route_open_principal_attested(frame, consumer_identity, attested)
4196 }
4197
4198 fn route_open_principal_attested(
4202 &self,
4203 frame: &Frame,
4204 consumer_identity: Option<ConsumerIdentity>,
4205 attested: Option<Principal>,
4206 ) -> Result<Result<Principal, Frame>, RouterError> {
4207 if let Some(principal) = attested {
4208 return Ok(Ok(principal));
4209 }
4210 let Some(consumer_identity) = consumer_identity else {
4211 return Ok(Ok(Principal::Direct));
4212 };
4213
4214 Ok(Err(control_error_frame(
4215 frame,
4216 "bad_consumer_identity",
4217 format!(
4218 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4219 consumer_identity.module_id
4220 ),
4221 )?))
4222 }
4223
4224 #[allow(clippy::too_many_arguments)]
4231 fn route_open_refusal_frame(
4232 &self,
4233 ctx: &RouteCtx,
4234 caller: &RouteOpenCaller,
4235 frame: &Frame,
4236 module_id: &str,
4237 reason: &'static str,
4238 code: &'static str,
4239 message: impl Into<String>,
4240 ) -> Result<Frame, RouterError> {
4241 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4242 control_error_frame(frame, code, message.into())
4243 }
4244
4245 #[allow(clippy::too_many_arguments)]
4254 fn route_bind_rejection(
4255 &self,
4256 ctx: &RouteCtx,
4257 caller: &RouteOpenCaller,
4258 frame: &Frame,
4259 target_module_id: &str,
4260 body: ErrorBody,
4261 daemon_authored: bool,
4262 ) -> Result<Vec<Frame>, RouterError> {
4263 if daemon_authored {
4269 let scope_code = match body.code.as_str() {
4270 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4271 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4272 _ => None,
4273 };
4274 if let Some(code) = scope_code {
4275 self.observe_route_open_refusal(
4276 ctx,
4277 caller,
4278 target_module_id,
4279 "scope_changed_before_commit",
4280 code,
4281 );
4282 return Ok(vec![control_error_body_frame(frame, body)?]);
4283 }
4284 }
4285 self.counters
4286 .increment_route_open_refused("module_rejected");
4287 info!(
4288 target: "control",
4289 code = "module_rejected",
4290 module_code = ?body.code,
4291 module_id = ?target_module_id,
4292 connection_id = ctx.connection_id.get(),
4293 principal = %caller.principal,
4294 scope = %caller.scope,
4295 harness = %caller.harness,
4296 "route.open refused"
4297 );
4298 let refusal = if daemon_authored {
4299 control_error_body_frame(frame, body)?
4300 } else {
4301 control_relayed_error_body_frame(frame, body)?
4302 };
4303 Ok(vec![refusal])
4304 }
4305
4306 #[allow(clippy::too_many_arguments)]
4324 fn route_open_breaker_refusal_frame(
4325 &self,
4326 ctx: &RouteCtx,
4327 caller: &RouteOpenCaller,
4328 frame: &Frame,
4329 module_id: &str,
4330 consecutive_timeouts: u32,
4331 retry_in: Duration,
4332 probe_in_flight: bool,
4333 ) -> Result<Frame, RouterError> {
4334 self.counters
4335 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4336 debug!(
4337 target: "control",
4338 code = "module_timeout",
4339 module_id = ?module_id,
4340 connection_id = ctx.connection_id.get(),
4341 principal = %caller.principal,
4342 scope = %caller.scope,
4343 harness = %caller.harness,
4344 consecutive_timeouts,
4345 retry_in_ms = retry_in.as_millis() as u64,
4346 probe_in_flight,
4347 "route.open refused by open bind-relay breaker"
4348 );
4349 let detail = if probe_in_flight {
4356 "one test route is already being tried; retry once it settles".to_string()
4357 } else {
4358 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4359 };
4360 control_error_frame(
4361 frame,
4362 "module_timeout",
4363 format!(
4364 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4365 timed out in a row); {detail}; its established routes are unaffected"
4366 ),
4367 )
4368 }
4369
4370 fn observe_route_open_refusal(
4383 &self,
4384 ctx: &RouteCtx,
4385 caller: &RouteOpenCaller,
4386 module_id: &str,
4387 reason: &'static str,
4388 code: &'static str,
4389 ) {
4390 self.counters.increment_route_open_refused(code);
4391 info!(
4392 target: "control",
4393 code,
4394 reason,
4395 module_id = ?module_id,
4396 connection_id = ctx.connection_id.get(),
4397 principal = %caller.principal,
4398 scope = %caller.scope,
4399 harness = %caller.harness,
4400 "route.open refused"
4401 );
4402 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4403 self.route_outages.record_not_serving(module_id, reason);
4404 }
4405 }
4406
4407 fn observe_route_open_accept(
4452 &self,
4453 ctx: &RouteCtx,
4454 module_id: &str,
4455 principal: &str,
4456 scope: &str,
4457 ) {
4458 self.route_outages.record_accepted(module_id);
4459 self.counters.increment_route_open_accepted(principal);
4460 info!(
4461 target: "control",
4462 principal,
4463 module_id,
4464 connection_id = ctx.connection_id.get(),
4465 scope = %scope,
4466 "route.open accepted"
4467 );
4468 }
4469
4470 fn supervised_absent_route_open_refusal_frame(
4471 &self,
4472 ctx: &RouteCtx,
4473 caller: &RouteOpenCaller,
4474 frame: &Frame,
4475 module_id: &str,
4476 code: &'static str,
4477 status: &crate::supervise::ModuleStatus,
4478 ) -> Result<Frame, RouterError> {
4479 self.counters.increment_route_open_refused(code);
4480 info!(
4481 target: "control",
4482 code,
4483 reason = "supervised_not_registered",
4484 module_id = ?module_id,
4485 connection_id = ctx.connection_id.get(),
4486 principal = %caller.principal,
4487 scope = %caller.scope,
4488 harness = %caller.harness,
4489 state = %status.state,
4490 enabled = status.enabled,
4491 live = status.live,
4492 "route.open refused"
4493 );
4494 self.route_outages
4498 .record_not_serving(module_id, "supervised_not_registered");
4499 let message = if let Some(pid) = status.waiting_for_previous_pid {
4500 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4501 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4502 })
4503 } else {
4504 format!(
4505 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4506 status.state, status.enabled, status.live
4507 )
4508 };
4509 control_error_frame(frame, code, message)
4510 }
4511
4512 async fn handle_route_open(
4513 &self,
4514 ctx: &RouteCtx,
4515 frame: Frame,
4516 request: RouteOpenRequest,
4517 timings: &mut RouteOpenTimings,
4518 ) -> Result<Vec<Frame>, RouterError> {
4519 let RouteOpenRequest {
4520 target,
4521 mut identity,
4522 consumer_identity,
4523 consumer_capabilities,
4524 role_versions,
4525 admission_facts,
4526 scope,
4527 } = request;
4528 let target_module_id = target_module_id(&target).to_string();
4529 timings.module_id.clone_from(&target_module_id);
4530 if self
4531 .registry
4532 .get_module_by_connection(ctx.connection_id)
4533 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4534 .is_some()
4535 {
4536 return Ok(vec![control_error_frame(
4537 &frame,
4538 "invalid_request",
4539 "module connections cannot open client routes",
4540 )?]);
4541 }
4542 debug!(
4543 connection_id = ctx.connection_id.get(),
4544 corr = frame.header.corr,
4545 module_id = %target_module_id,
4546 "handling route.open"
4547 );
4548
4549 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4554 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4555
4556 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4562 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4563 self.observe_route_open_refusal(
4564 ctx,
4565 &caller,
4566 &target_module_id,
4567 "invalid_role_versions",
4568 error_codes::INVALID_REQUEST,
4569 );
4570 return Ok(vec![control_error_body_frame(
4571 &frame,
4572 ErrorBody {
4573 code: error_codes::INVALID_REQUEST.to_string(),
4574 message: error.to_string(),
4575 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4576 },
4577 )?]);
4578 }
4579
4580 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4581 if status.waiting_for_previous_pid.is_some() {
4582 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4583 ctx,
4584 &caller,
4585 &frame,
4586 &target_module_id,
4587 "module_warming",
4588 &status,
4589 )?]);
4590 }
4591 }
4592
4593 let Some(registration) = self
4612 .registry
4613 .get_module(&target_module_id)
4614 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4615 else {
4616 if let Some((status, warming)) =
4617 self.supervisor_status(&target_module_id, frame.header.corr)?
4618 {
4619 if status.protocol == ModuleProtocol::None {
4626 return Ok(vec![self.route_open_refusal_frame(
4627 ctx,
4628 &caller,
4629 &frame,
4630 &target_module_id,
4631 "protocol_none",
4632 error_codes::MODULE_NO_PROTOCOL,
4633 format!(
4634 "module_id '{target_module_id}' is declared protocol: none; \
4635 it speaks no subc wire and serves no routes"
4636 ),
4637 )?]);
4638 }
4639 let code = if warming {
4640 "module_warming"
4641 } else {
4642 "target_unavailable"
4643 };
4644 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4645 ctx,
4646 &caller,
4647 &frame,
4648 &target_module_id,
4649 code,
4650 &status,
4651 )?]);
4652 }
4653 if let Some(removed_ago_ms) =
4654 self.supervisor.removal_tombstone_age_ms(&target_module_id)
4655 {
4656 return Ok(vec![self.route_open_refusal_frame(
4657 ctx,
4658 &caller,
4659 &frame,
4660 &target_module_id,
4661 "removed",
4662 error_codes::MODULE_REMOVED,
4663 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
4664 )?]);
4665 }
4666 return Ok(vec![self.route_open_refusal_frame(
4667 ctx,
4668 &caller,
4669 &frame,
4670 &target_module_id,
4671 "not_registered",
4672 error_codes::UNKNOWN_MODULE,
4673 format!("module_id '{target_module_id}' is not registered"),
4674 )?]);
4675 };
4676
4677 if !registration.ready {
4681 self.counters
4682 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
4683 info!(
4684 target: "control",
4685 code = error_codes::MODULE_WARMING,
4686 module_id = ?target_module_id,
4687 connection_id = ctx.connection_id.get(),
4688 principal = %caller.principal,
4689 scope = %caller.scope,
4690 harness = %caller.harness,
4691 reason = "declared_not_ready",
4692 "route.open refused"
4693 );
4694 self.route_outages
4697 .record_not_serving(&target_module_id, "declared_not_ready");
4698 return Ok(vec![control_error_body_frame(
4699 &frame,
4700 ErrorBody {
4701 code: error_codes::MODULE_WARMING.to_string(),
4702 message: format!(
4703 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
4704 ),
4705 detail: Some(serde_json::json!({
4706 "reason": "declared_not_ready"
4707 })),
4708 },
4709 )?]);
4710 }
4711
4712 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
4731 self.counters
4732 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
4733 info!(
4734 target: "control",
4735 code = error_codes::MODULE_WARMING,
4736 module_id = ?target_module_id,
4737 connection_id = ctx.connection_id.get(),
4738 principal = %caller.principal,
4739 scope = %caller.scope,
4740 harness = %caller.harness,
4741 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4742 capability = %capability,
4743 "route.open refused"
4744 );
4745 return Ok(vec![control_error_body_frame(
4746 &frame,
4747 ErrorBody {
4748 code: error_codes::MODULE_WARMING.to_string(),
4749 message: format!(
4750 "module_id '{target_module_id}' requires capability '{capability}', \
4751 which no registered module provides; retry"
4752 ),
4753 detail: Some(serde_json::json!({
4754 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
4755 "capability": capability,
4756 })),
4757 },
4758 )?]);
4759 }
4760
4761 if !target_has_required_role(&target, ®istration.manifest.provides) {
4762 return Ok(vec![self.route_open_refusal_frame(
4763 ctx,
4764 &caller,
4765 &frame,
4766 &target_module_id,
4767 "role_not_provided",
4768 "target_unavailable",
4769 format!("module_id '{target_module_id}' does not provide the requested target"),
4770 )?]);
4771 }
4772
4773 if registration.state != ChannelState::Active {
4774 return Ok(vec![self.route_open_refusal_frame(
4775 ctx,
4776 &caller,
4777 &frame,
4778 &target_module_id,
4779 "registration_not_active",
4780 "target_unavailable",
4781 format!("module_id '{target_module_id}' is not active"),
4782 )?]);
4783 }
4784
4785 if self
4786 .forwarding
4787 .module_is_draining(&target_module_id)
4788 .map_err(RouterError::Forwarding)?
4789 {
4790 return Ok(vec![self.route_open_refusal_frame(
4791 ctx,
4792 &caller,
4793 &frame,
4794 &target_module_id,
4795 "reloading",
4796 "module_reloading",
4797 format!("module_id '{target_module_id}' is reloading"),
4798 )?]);
4799 }
4800
4801 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
4802 process_liveness.process_live(&target_module_id) == Some(false)
4803 }) {
4804 if process_liveness.process_replacing(&target_module_id) {
4811 return Ok(vec![self.route_open_refusal_frame(
4812 ctx,
4813 &caller,
4814 &frame,
4815 &target_module_id,
4816 "reloading",
4817 "module_reloading",
4818 format!("module_id '{target_module_id}' is reloading"),
4819 )?]);
4820 }
4821 return Ok(vec![self.route_open_refusal_frame(
4822 ctx,
4823 &caller,
4824 &frame,
4825 &target_module_id,
4826 "supervisor_not_live",
4827 "target_unavailable",
4828 format!("module_id '{target_module_id}' is not live"),
4829 )?]);
4830 }
4831
4832 if !self
4833 .forwarding
4834 .has_live_module_connection(&target_module_id)
4835 .map_err(RouterError::Forwarding)?
4836 {
4837 return Ok(vec![self.route_open_refusal_frame(
4838 ctx,
4839 &caller,
4840 &frame,
4841 &target_module_id,
4842 "no_forwarding_connection",
4843 "target_unavailable",
4844 format!("module_id '{target_module_id}' has no live forwarding connection"),
4845 )?]);
4846 }
4847
4848 if let Some(error) =
4849 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
4850 {
4851 self.observe_route_open_refusal(
4852 ctx,
4853 &caller,
4854 &target_module_id,
4855 "op_not_allowed",
4856 "op_not_allowed",
4857 );
4858 return Ok(vec![error]);
4859 }
4860
4861 let principal =
4862 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
4863 Ok(principal) => principal,
4864 Err(error) => {
4865 self.observe_route_open_refusal(
4866 ctx,
4867 &caller,
4868 &target_module_id,
4869 "bad_consumer_identity",
4870 "bad_consumer_identity",
4871 );
4872 return Ok(vec![error]);
4873 }
4874 };
4875 timings.principal = principal_label(&principal);
4876
4877 let build_class = identity.build_class.take().filter(|build_class| {
4880 if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
4881 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
4882 } else {
4883 if self
4884 .build_class_warned
4885 .lock()
4886 .unwrap_or_else(|p| p.into_inner())
4887 .insert(ctx.connection_id)
4888 {
4889 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
4890 "ignoring build_class from a non-Callosum opener");
4891 }
4892 false
4893 }
4894 });
4895
4896 if let Principal::Reserved {
4900 module_id: opening_module_id,
4901 } = &principal
4902 {
4903 if let Some(opening_registration) = self
4904 .registry
4905 .get_module(opening_module_id)
4906 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4907 {
4908 if let Some(capability) =
4909 denied_capability(&opening_registration.manifest, ®istration.manifest)
4910 {
4911 warn!(
4912 opening_module_id,
4913 target_module_id,
4914 capability,
4915 "refusing route.open because an attested capability deny edge matches"
4916 );
4917 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4918 &frame,
4919 &target_module_id,
4920 "capability_deny_edge",
4921 "capability_forbidden",
4922 format!(
4923 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
4924 ),
4925 )?]);
4926 }
4927 }
4928 }
4929
4930 if admission_facts.is_some() {
4931 let carrier_matches = matches!(
4932 &principal,
4933 Principal::Reserved { module_id }
4934 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
4935 );
4936 if !carrier_matches {
4937 return Ok(vec![self.route_open_refusal_frame(
4938 ctx,
4939 &caller,
4940 &frame,
4941 &target_module_id,
4942 "admission_facts_carrier_not_permitted",
4943 "admission_facts_not_permitted",
4944 "admission facts may only be carried by the configured reserved module",
4945 )?]);
4946 }
4947
4948 let target_allowed = self
4949 .admission_facts_targets
4950 .as_ref()
4951 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
4952 if !target_allowed {
4953 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
4954 &frame,
4955 &target_module_id,
4956 "admission_facts_target_not_listed",
4957 "admission_facts_target_not_allowed",
4958 format!(
4959 "admission facts are not permitted for target module_id '{target_module_id}'"
4960 ),
4961 )?]);
4962 }
4963
4964 }
4968
4969 let (bound_scope, mut scope_stamp) = match &scope {
4975 None => (None, None),
4976 Some(selector) => {
4977 let owner_configured = match &selector.owner {
4978 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
4984 _ => false,
4985 };
4986 let mut scopes = self
4987 .scopes
4988 .write()
4989 .unwrap_or_else(|poisoned| poisoned.into_inner());
4990 let admitted = scopes
4991 .admit(&principal, &target_module_id, selector, owner_configured)
4992 .and_then(|admission| {
4993 crate::scopes::check_target_flow_support(
4994 &admission.stamp,
4995 &target_module_id,
4996 registration.manifest.capabilities.as_ref(),
4997 )?;
4998 crate::scopes::check_target_agent_run_support(
4999 &admission.stamp,
5000 &target_module_id,
5001 registration.manifest.capabilities.as_ref(),
5002 )?;
5003 let bound = scopes.reserve_bind(
5004 &admission,
5005 &principal,
5006 ctx.connection_id,
5007 selector.hold_presence,
5008 );
5009 Ok((admission, bound))
5010 });
5011 match admitted {
5012 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
5013 Err(refusal) => {
5014 return Ok(vec![self.route_open_refusal_frame(
5015 ctx,
5016 &caller,
5017 &frame,
5018 &target_module_id,
5019 refusal.code,
5020 refusal.code,
5021 refusal.message,
5022 )?]);
5023 }
5024 }
5025 }
5026 };
5027 if let Some(stamp) = &scope_stamp {
5030 caller.scope = route_open_scope_label(stamp);
5031 }
5032
5033 timings.enter(RouteOpenPhase::ProjectRoot);
5055 let project_root = match ProjectRootId::from_path_allowing_missing(&identity.project_root) {
5056 Ok(project_root) => project_root,
5057 Err(err) => {
5058 return Ok(vec![control_error_frame(
5059 &frame,
5060 "invalid_project_root",
5061 err.to_string(),
5062 )?])
5063 }
5064 };
5065 timings.enter(RouteOpenPhase::Admission);
5066 identity.project_root = project_root.as_path().to_path_buf();
5067
5068 let _concurrency_guard = match self
5079 .route_bind_concurrency
5080 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5081 {
5082 Ok(guard) => guard,
5083 Err(in_flight) => {
5084 return Ok(vec![self.route_open_target_capacity_refusal(
5085 ctx,
5086 &caller,
5087 &frame,
5088 &target_module_id,
5089 in_flight,
5090 )?]);
5091 }
5092 };
5093
5094 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5097 RouteBindAdmission::Admitted { guard, probe } => {
5098 if probe {
5099 info!(
5100 module_id = %target_module_id,
5101 connection_id = ctx.connection_id.get(),
5102 "route.bind breaker half-open: admitting one probe"
5103 );
5104 }
5105 guard
5106 }
5107 RouteBindAdmission::Refused {
5108 consecutive_timeouts,
5109 retry_in,
5110 probe_in_flight,
5111 } => {
5112 return Ok(vec![self.route_open_breaker_refusal_frame(
5113 ctx,
5114 &caller,
5115 &frame,
5116 &target_module_id,
5117 consecutive_timeouts,
5118 retry_in,
5119 probe_in_flight,
5120 )?]);
5121 }
5122 };
5123
5124 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5129 let relay_deadline = Instant::now() + route_bind_relay_timeout;
5130 let operator_app = self
5131 .operator_apps
5132 .lock()
5133 .unwrap_or_else(|p| p.into_inner())
5134 .get(&ctx.connection_id)
5135 .cloned();
5136 timings.enter(RouteOpenPhase::EgressReservation);
5137 let pending = match self
5138 .forwarding
5139 .begin_route_bind_relay_for(
5140 ctx.connection_id,
5141 ctx.egress.clone(),
5142 response_version(&frame),
5143 frame.header.corr,
5144 &target_module_id,
5145 principal.clone(),
5146 bound_scope,
5147 Some(project_root),
5148 identity.harness.clone(),
5149 build_class,
5150 operator_app.clone(),
5151 relay_deadline,
5152 )
5153 .await
5154 {
5155 Ok(pending) => pending,
5156 Err(err) => {
5157 return Ok(vec![self.route_open_refusal_frame(
5158 ctx,
5159 &caller,
5160 &frame,
5161 &target_module_id,
5162 "relay_reservation_failed",
5163 forwarding_error_code(&err),
5164 err.to_string(),
5165 )?])
5166 }
5167 };
5168 timings.enter(RouteOpenPhase::Admission);
5169 let crate::forwarding::PendingRouteBindRelay {
5170 endpoint,
5171 module_sink,
5172 negotiated_ver,
5173 client_channel,
5174 client_epoch,
5175 module_channel,
5176 module_epoch,
5177 corr: relay_corr,
5178 receiver,
5179 } = pending;
5180 let mut reservation =
5181 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5182
5183 let relay_registration = self
5188 .registry
5189 .get_module_by_connection(endpoint.connection_id)
5190 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5191 let relay_capabilities = relay_registration
5192 .as_ref()
5193 .and_then(|registration| registration.manifest.capabilities.as_ref());
5194 if let Some(selector) = &scope {
5195 if let Err(refusal) = self
5198 .scopes
5199 .read()
5200 .unwrap_or_else(|poisoned| poisoned.into_inner())
5201 .check_bind_proof(&principal, selector)
5202 {
5203 reservation.release_and_disarm();
5204 return Ok(vec![self.route_open_refusal_frame(
5205 ctx,
5206 &caller,
5207 &frame,
5208 &target_module_id,
5209 refusal.code,
5210 refusal.code,
5211 refusal.message,
5212 )?]);
5213 }
5214 }
5215 if let Some(stamp) = scope_stamp
5216 .as_ref()
5217 .filter(|stamp| stamp.attributes.flow_id.is_some() || stamp.attributes.run_id.is_some())
5218 {
5219 if let Err(refusal) = crate::scopes::check_target_flow_support(
5220 stamp,
5221 &target_module_id,
5222 relay_capabilities,
5223 )
5224 .and_then(|()| {
5225 crate::scopes::check_target_agent_run_support(
5226 stamp,
5227 &target_module_id,
5228 relay_capabilities,
5229 )
5230 }) {
5231 reservation.release_and_disarm();
5232 return Ok(vec![self.route_open_refusal_frame(
5233 ctx,
5234 &caller,
5235 &frame,
5236 &target_module_id,
5237 refusal.code,
5238 refusal.code,
5239 refusal.message,
5240 )?]);
5241 }
5242 }
5243
5244 if !relay_capabilities.is_some_and(|caps| {
5247 caps.provides
5248 .iter()
5249 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5250 }) {
5251 if let Some(stamp) = scope_stamp.as_mut() {
5252 stamp.workdir = None;
5253 stamp.project_id = None;
5254 }
5255 }
5256 let worker_run = scope_stamp
5261 .as_ref()
5262 .is_some_and(crate::scopes::is_worker_run)
5263 && relay_capabilities.is_some_and(|caps| {
5264 caps.provides
5265 .iter()
5266 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5267 });
5268 if !worker_run
5269 && !relay_capabilities.is_some_and(|caps| {
5270 caps.provides
5271 .iter()
5272 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5273 })
5274 {
5275 if let Some(stamp) = scope_stamp.as_mut() {
5276 stamp.attributes.on_behalf_of_agent = None;
5277 }
5278 }
5279 if !relay_capabilities.is_some_and(|caps| {
5280 caps.provides
5281 .iter()
5282 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5283 }) {
5284 if let Some(stamp) = scope_stamp.as_mut() {
5285 stamp.attributes.flow_version = None;
5286 stamp.attributes.flow_code_hash = None;
5287 stamp.attributes.flow_install_id = None;
5288 }
5289 }
5290 let call_mode = if worker_run
5291 || relay_capabilities
5292 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5293 {
5294 Some(derive_call_mode(scope_stamp.as_ref()))
5295 } else {
5296 if let Some(stamp) = scope_stamp.as_mut() {
5297 stamp.expires_at_ms = None;
5298 }
5299 None
5300 };
5301
5302 debug!(
5303 connection_id = ctx.connection_id.get(),
5304 client_channel,
5305 client_epoch,
5306 module_channel,
5307 module_epoch,
5308 "reserved route handle pair"
5309 );
5310 let route_principal_label = principal_label(&principal);
5313 let relay = ModuleControlRequest::RouteBind {
5314 route_channel: module_channel,
5315 epoch: module_epoch,
5316 target,
5317 identity,
5318 principal: Some(principal),
5319 consumer_capabilities,
5320 role_versions,
5321 admission_facts,
5322 scope: scope_stamp,
5323 call_mode,
5324 };
5325 let peer = RouteBindPeer::new()
5326 .with_observed_peer(
5327 relay_capabilities
5328 .is_some_and(|caps| {
5329 caps.provides
5330 .iter()
5331 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5332 })
5333 .then(|| {
5334 self.observed_peers
5335 .lock()
5336 .unwrap_or_else(|p| p.into_inner())
5337 .get(&ctx.connection_id)
5338 .cloned()
5339 .unwrap_or_else(ObservedPeer::unobserved)
5340 }),
5341 )
5342 .with_operator_app(
5343 operator_app
5344 .filter(|_| {
5345 self.operator_app_policy.macos_enabled
5346 && relay_capabilities.is_some_and(|caps| {
5347 caps.provides
5348 .iter()
5349 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5350 })
5351 })
5352 .map(|stamp| stamp.app),
5353 );
5354 let relay_body = encode_route_bind(&relay, &peer).map_err(|err| {
5355 RouterError::backend(
5356 0,
5357 frame.header.corr,
5358 format!("failed to encode route.bind request: {err}"),
5359 )
5360 })?;
5361 let relay_frame = Frame::build_with_version(
5362 negotiated_ver,
5363 FrameType::Request,
5364 control_flags(),
5365 0,
5366 0,
5367 relay_corr,
5368 relay_body,
5369 )
5370 .map_err(RouterError::FrameBuild)?;
5371
5372 timings.enter(RouteOpenPhase::BindSend);
5373 let sent = module_sink.send(relay_frame).await;
5374 timings.enter(RouteOpenPhase::Admission);
5375 if let Err(err) = sent {
5376 reservation.release_and_disarm();
5377 return Ok(vec![self.route_open_refusal_frame(
5378 ctx,
5379 &caller,
5380 &frame,
5381 &target_module_id,
5382 "relay_send_failed",
5383 "target_unavailable",
5384 err.to_string(),
5385 )?]);
5386 }
5387
5388 #[cfg(test)]
5389 if let Some(hold) = &self.route_bind_relay_sent_hold {
5390 hold.wait().await;
5394 hold.wait().await;
5395 }
5396
5397 if self
5402 .forwarding
5403 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5404 .map_err(RouterError::Forwarding)?
5405 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5406 {
5407 self.send_abandoned_route_bind_goodbye(
5408 &module_sink,
5409 negotiated_ver,
5410 module_channel,
5411 module_epoch,
5412 );
5413 }
5414
5415 timings.enter(RouteOpenPhase::BindAckWait);
5416 let outcome = timeout_at(relay_deadline, receiver).await;
5417 timings.enter(RouteOpenPhase::Admission);
5418 match outcome {
5419 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5420 reservation.disarm();
5421 if breaker.record_accepted() {
5422 info!(
5423 module_id = %target_module_id,
5424 "route.bind breaker closed: the probe was accepted"
5425 );
5426 }
5427 self.observe_route_open_accept(
5428 ctx,
5429 &target_module_id,
5430 &route_principal_label,
5431 &caller.scope,
5432 );
5433 Ok(Vec::new())
5434 }
5435 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5436 reservation.release_and_disarm();
5437 breaker.record_inconclusive();
5441 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5442 }
5443 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5444 reservation.release_and_disarm();
5445 breaker.record_inconclusive();
5446 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5447 }
5448 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5449 reservation.release_and_disarm();
5450 breaker.record_inconclusive();
5451 tracing::warn!(
5458 module_id = %target_module_id,
5459 "route.bind relay abandoned: {message}"
5460 );
5461 Ok(vec![self.route_open_refusal_frame(
5462 ctx,
5463 &caller,
5464 &frame,
5465 &target_module_id,
5466 "relay_abandoned",
5467 "target_unavailable",
5468 message,
5469 )?])
5470 }
5471 Ok(Err(_)) => {
5472 reservation.release_and_disarm();
5473 breaker.record_inconclusive();
5474 Ok(vec![self.route_open_refusal_frame(
5475 ctx,
5476 &caller,
5477 &frame,
5478 &target_module_id,
5479 "relay_waiter_canceled",
5480 "target_unavailable",
5481 "route.bind relay waiter was canceled before the module responded",
5482 )?])
5483 }
5484 Err(_) => {
5485 reservation.release_and_disarm();
5486 if let Some(opened) = breaker.record_timeout(
5490 self.route_bind_breaker_threshold,
5491 self.route_bind_breaker_cooldown,
5492 ) {
5493 warn!(
5494 module_id = %target_module_id,
5495 consecutive_timeouts = opened.consecutive_timeouts,
5496 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5497 reopened_after_probe = opened.reopened_after_probe,
5498 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5499 );
5500 }
5501 tracing::warn!(
5509 module_id = %target_module_id,
5510 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5511 "route.bind relay timed out: module did not ack within budget"
5512 );
5513 Ok(vec![self.route_open_refusal_frame(
5514 ctx,
5515 &caller,
5516 &frame,
5517 &target_module_id,
5518 "relay_timed_out",
5519 "module_timeout",
5520 format!(
5521 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5522 route_bind_relay_timeout
5523 ),
5524 )?])
5525 }
5526 }
5527 }
5528
5529 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5530 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5531 snapshot: self.supervisor.spawn_snapshot(),
5532 };
5533 Ok(vec![control_response_body_frame(
5534 &frame,
5535 &response,
5536 "ClientControlResponse::SupervisorSpawnSnapshot",
5537 )?])
5538 }
5539
5540 fn handle_supervisor_spawn_subscribe(
5541 &self,
5542 ctx: &RouteCtx,
5543 frame: Frame,
5544 since: Option<SpawnCursor>,
5545 ) -> Result<Vec<Frame>, RouterError> {
5546 match self.supervisor.subscribe_spawns(
5547 ctx.connection_id,
5548 frame.header.corr,
5549 response_version(&frame),
5550 since,
5551 ctx.egress.clone(),
5552 ) {
5553 Ok(()) => Ok(Vec::new()),
5554 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5555 Ok(vec![control_error_body_frame(
5556 &frame,
5557 ErrorBody {
5558 code: "spawn_cursor_incarnation_mismatch".to_string(),
5559 message: "spawn cursor belongs to a different daemon incarnation"
5560 .to_string(),
5561 detail: Some(serde_json::json!({
5562 "current_daemon_incarnation": current
5563 })),
5564 },
5565 )?])
5566 }
5567 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5568 &frame,
5569 ErrorBody {
5570 code: "spawn_cursor_too_old".to_string(),
5571 message: "spawn cursor predates the retained event ring".to_string(),
5572 detail: Some(serde_json::json!({
5573 "oldest_retained_cursor": oldest
5574 })),
5575 },
5576 )?]),
5577 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5578 0,
5579 frame.header.corr,
5580 format!("failed to open supervisor spawn subscription: {error}"),
5581 )),
5582 }
5583 }
5584
5585 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5586 let generation = self
5587 .registry
5588 .generation()
5589 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5590 let mut modules = Vec::new();
5591 for module in self.supervisor.list() {
5592 let status = module.status_for_control("list").map_err(|err| {
5593 RouterError::backend(
5594 0,
5595 frame.header.corr,
5596 format!("failed to read supervisor status: {err}"),
5597 )
5598 })?;
5599 let (configured, _) = module.configuration().map_err(|err| {
5600 RouterError::backend(
5601 0,
5602 frame.header.corr,
5603 format!("failed to read module configuration: {err}"),
5604 )
5605 })?;
5606 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
5607 RouterError::backend(
5608 0,
5609 frame.header.corr,
5610 format!("failed to read retry time: {err}"),
5611 )
5612 })?;
5613 let image = module.running_image_agreement().await;
5615 let resources = Some(module.child_resource_usage());
5618 let mut verdict =
5619 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
5620 verdict.spec = Some(launch_spec_agreement(
5621 &configured,
5622 status.process_alive,
5623 status.launch_spec_digest.as_ref(),
5624 ));
5625 let pending_reload = Some(verdict);
5626 let configured_enabled = self
5627 .capability_evaluator
5628 .configured_enabled(&status.module_id);
5629 modules.push(
5632 SupervisorEntry::new(
5633 status.module_id,
5634 status.state.to_string(),
5635 status.enabled,
5636 status.live,
5637 status.health.status,
5638 )
5639 .with_configured_enabled(configured_enabled)
5640 .with_next_retry_at_ms(next_retry_at_ms)
5641 .with_launch_nonce_env(Some(!cfg!(unix)))
5642 .with_launch_nonce_source(status.launch_nonce_source)
5643 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
5644 .with_protocol(status.protocol)
5645 .with_pending_reload(pending_reload)
5646 .with_last_probe_ms(status.health.last_probe_ms)
5647 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
5648 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
5649 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
5650 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
5651 .with_restart_count(Some(status.restart_count))
5652 .with_max_restarts(Some(status.max_restarts))
5653 .with_lifetime_restarts(Some(status.lifetime_restarts))
5654 .with_spawn_generation(Some(status.spawn_generation))
5655 .with_restart_window_secs(Some(status.restart_window.as_secs()))
5656 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
5657 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
5658 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
5659 .with_resources(resources),
5660 );
5661 }
5662 let response = ClientControlResponse::SupervisorList {
5663 generation,
5664 modules,
5665 };
5666 Ok(vec![control_response_body_frame(
5667 &frame,
5668 &response,
5669 "ClientControlResponse::SupervisorList",
5670 )?])
5671 }
5672
5673 fn handle_supervisor_stderr_tail(
5674 &self,
5675 frame: Frame,
5676 module_id: String,
5677 max_lines: Option<u32>,
5678 max_bytes: Option<u32>,
5679 ) -> Result<Vec<Frame>, RouterError> {
5680 let Some(module) = self.supervisor.get(&module_id) else {
5681 return Ok(vec![control_error_frame(
5682 &frame,
5683 "unknown_module",
5684 format!("module_id '{module_id}' is not supervised"),
5685 )?]);
5686 };
5687
5688 let snapshot = module.stderr_tail(
5689 max_lines.map(|value| value as usize),
5690 max_bytes.map(|value| value as usize),
5691 );
5692
5693 let response = ClientControlResponse::SupervisorStderrTail {
5694 module_id,
5695 tail: StderrTail {
5696 capture: match snapshot.capture {
5697 CaptureState::Captured => StderrCaptureState::Captured,
5698 CaptureState::Incomplete { reason } => {
5699 StderrCaptureState::Incomplete { reason }
5700 }
5701 CaptureState::NotCaptured { reason } => {
5702 StderrCaptureState::NotCaptured { reason }
5703 }
5704 },
5705 entries: snapshot
5706 .entries
5707 .into_iter()
5708 .map(|entry| match entry {
5709 TailEntry::Line {
5710 text,
5711 truncated,
5712 at_ms,
5713 } => StderrTailEntry::Line {
5714 text,
5715 truncated,
5716 at_ms,
5717 },
5718 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
5719 })
5720 .collect(),
5721 dropped_lines: snapshot.dropped_lines,
5722 },
5723 };
5724 Ok(vec![control_response_body_frame(
5725 &frame,
5726 &response,
5727 "ClientControlResponse::SupervisorStderrTail",
5728 )?])
5729 }
5730
5731 async fn handle_supervisor_terminals(
5732 &self,
5733 frame: Frame,
5734 module_id: String,
5735 ) -> Result<Vec<Frame>, RouterError> {
5736 let Some(module) = self.supervisor.get(&module_id) else {
5737 return Ok(vec![control_error_frame(
5738 &frame,
5739 "unknown_module",
5740 format!("module_id '{module_id}' is not supervised"),
5741 )?]);
5742 };
5743
5744 let terminals = module
5747 .read_durable_terminal_history()
5748 .await
5749 .map_err(|error| {
5750 RouterError::backend(
5751 0,
5752 frame.header.corr,
5753 format!("failed to read terminal history: {error}"),
5754 )
5755 })?;
5756 let response = ClientControlResponse::SupervisorTerminals {
5757 module_id,
5758 terminals,
5759 };
5760 Ok(vec![control_response_body_frame(
5761 &frame,
5762 &response,
5763 "ClientControlResponse::SupervisorTerminals",
5764 )?])
5765 }
5766
5767 fn handle_supervisor_routes(
5768 &self,
5769 frame: Frame,
5770 module_id: Option<String>,
5771 ) -> Result<Vec<Frame>, RouterError> {
5772 let modules = self
5773 .forwarding
5774 .route_census(module_id.as_deref())
5775 .map_err(RouterError::Forwarding)?
5776 .into_iter()
5777 .map(|(module_id, routes)| SupervisorRouteModule {
5778 module_id,
5779 routes: routes
5780 .into_iter()
5781 .map(|route| SupervisorRoute {
5782 consumer: match route.principal {
5783 Principal::Reserved { module_id } => {
5784 SupervisorRouteConsumer::Reserved { module_id }
5785 }
5786 Principal::Direct | Principal::Unverified => {
5787 SupervisorRouteConsumer::Direct {
5788 connection_id: route.goodbye_target.connection_id.get(),
5789 }
5790 }
5791 },
5792 age_ms: Instant::now()
5793 .saturating_duration_since(route.bound_at)
5794 .as_millis()
5795 .try_into()
5796 .unwrap_or(u64::MAX),
5797 draining: route.draining,
5798 drain_reason: route.drain_reason,
5799 })
5800 .collect(),
5801 })
5802 .collect();
5803 let response = ClientControlResponse::SupervisorRoutes { modules };
5804 Ok(vec![control_response_body_frame(
5805 &frame,
5806 &response,
5807 "ClientControlResponse::SupervisorRoutes",
5808 )?])
5809 }
5810
5811 async fn handle_supervisor_provenance(
5812 &self,
5813 frame: Frame,
5814 module_id: Option<String>,
5815 ) -> Result<Vec<Frame>, RouterError> {
5816 let mut selected = if let Some(module_id) = module_id {
5817 let Some(module) = self.supervisor.get(&module_id) else {
5818 return Ok(vec![control_error_frame(
5819 &frame,
5820 "unknown_module",
5821 format!("module_id '{module_id}' is not supervised"),
5822 )?]);
5823 };
5824 vec![module]
5825 } else {
5826 self.supervisor.list()
5827 };
5828
5829 let mut modules = Vec::with_capacity(selected.len());
5830 for module in selected.drain(..) {
5831 let (status, observed_image) = module
5832 .status_and_running_image_agreement()
5833 .await
5834 .map_err(|err| {
5835 RouterError::backend(
5836 0,
5837 frame.header.corr,
5838 format!("failed to read supervisor status: {err}"),
5839 )
5840 })?;
5841 let module_declared = self
5842 .registry
5843 .get_module(&status.module_id)
5844 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5845 .and_then(|registration| registration.manifest.provenance)
5846 .map(|build| ModuleDeclaredProvenance::Reported { build })
5847 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
5848 #[cfg(test)]
5849 let running_image = match &self.provenance_probe_override {
5850 Some(result) => result.clone(),
5851 None => observed_image,
5852 };
5853 #[cfg(not(test))]
5854 let running_image = observed_image;
5855 modules.push(SupervisorModuleProvenance {
5856 module_id: status.module_id,
5857 module_declared,
5858 daemon_observed: SupervisorObservedProcess {
5859 launch_nonce_source: status.launch_nonce_source,
5860 pid: status.pid,
5861 spawned_at_ms: status.spawned_at_ms,
5862 spawned_from: status.spawned_from,
5863 running_image,
5864 },
5865 });
5866 }
5867 let daemon = SupervisorDaemonProvenance {
5868 daemon_build: self.daemon_provenance.build.clone(),
5869 daemon_observed: DaemonObservedProcess {
5870 pid: self.daemon_provenance.pid,
5871 started_at_ms: self
5872 .daemon_provenance
5873 .start_clock
5874 .map(|clock| clock.started_at_ms())
5875 .or(self.daemon_provenance.started_at_ms),
5876 running_image: self
5877 .daemon_provenance
5878 .probe
5879 .observe(
5880 self.daemon_provenance.pid,
5881 self.daemon_provenance.executable_path.as_deref(),
5882 self.daemon_provenance.executable_identity.clone(),
5883 self.daemon_provenance.process_start_time,
5884 )
5885 .await,
5886 },
5887 };
5888 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
5889 Ok(vec![control_response_body_frame(
5890 &frame,
5891 &response,
5892 "ClientControlResponse::SupervisorProvenance",
5893 )?])
5894 }
5895
5896 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5897 self.refresh_capability_requirements();
5898 let generation = self
5899 .registry
5900 .generation()
5901 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5902 let modules = self
5903 .supervisor
5904 .list()
5905 .into_iter()
5906 .map(|module| {
5907 let status = module.status_for_control("health").map_err(|err| {
5908 RouterError::backend(
5909 0,
5910 frame.header.corr,
5911 format!("failed to read supervisor health: {err}"),
5912 )
5913 })?;
5914 let module_id = status.module_id;
5915 let capability_detail = self
5916 .capability_evaluator
5917 .required_problem_detail(&module_id);
5918 Ok(SupervisorHealthEntry {
5919 module_id,
5920 status: status.health.status,
5921 detail: append_capability_problem_detail(
5922 status.health.detail,
5923 capability_detail,
5924 ),
5925 metrics: status.health.metrics,
5926 consecutive_failures: status.health.consecutive_failures,
5927 late_answer_count: status.health.late_answer_count,
5928 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
5929 last_action: status.health.last_action,
5930 last_action_ms: status.health.last_action_ms,
5931 last_probe_ms: status.health.last_probe_ms,
5932 })
5933 })
5934 .collect::<Result<Vec<_>, RouterError>>()?;
5935 let response = ClientControlResponse::SupervisorHealth {
5936 generation,
5937 modules,
5938 };
5939 Ok(vec![control_response_body_frame(
5940 &frame,
5941 &response,
5942 "ClientControlResponse::SupervisorHealth",
5943 )?])
5944 }
5945
5946 async fn handle_supervisor_restart(
5947 &self,
5948 frame: Frame,
5949 module_id: String,
5950 drain_timeout_ms: Option<u64>,
5951 requester: Principal,
5952 ) -> Result<Vec<Frame>, RouterError> {
5953 let operation_lock = self.supervisor.operation_lock();
5954 let _operation_guard = operation_lock.lock().await;
5955 let Some(module) = self.supervisor.get(&module_id) else {
5956 return Ok(vec![control_error_frame(
5957 &frame,
5958 "unknown_module",
5959 format!("module_id '{module_id}' is not supervised"),
5960 )?]);
5961 };
5962
5963 self.route_outages.mark_operator_action(&module_id);
5964 if let Err(err) = module
5965 .restart_requested(drain_timeout_ms, Some(requester))
5966 .await
5967 {
5968 self.route_outages
5969 .operator_action_ended_unrefused(&module_id);
5970 let (code, message) = match err {
5971 crate::supervise::SuperviseError::Disabled { .. } => {
5972 ("module_disabled", err.to_string())
5973 }
5974 crate::supervise::SuperviseError::SwapInProgress { .. } => {
5975 ("swap_in_progress", err.to_string())
5976 }
5977 _ => (
5978 "target_unavailable",
5979 format!("failed to restart module_id '{module_id}': {err}"),
5980 ),
5981 };
5982 return Ok(vec![control_error_frame(&frame, code, message)?]);
5983 }
5984
5985 let response = ClientControlResponse::SupervisorAck {
5986 module_id,
5987 applied: true,
5988 };
5989 Ok(vec![control_response_body_frame(
5990 &frame,
5991 &response,
5992 "ClientControlResponse::SupervisorAck",
5993 )?])
5994 }
5995
5996 async fn handle_supervisor_swap(
6000 &self,
6001 frame: Frame,
6002 module_id: String,
6003 ready_timeout_ms: Option<u64>,
6004 requester: Principal,
6005 ) -> Result<Vec<Frame>, RouterError> {
6006 let module = {
6013 let operation_lock = self.supervisor.operation_lock();
6014 let _operation_guard = operation_lock.lock().await;
6015 self.supervisor.get(&module_id)
6016 };
6017 let Some(module) = module else {
6018 return Ok(vec![control_error_frame(
6019 &frame,
6020 "unknown_module",
6021 format!("module_id '{module_id}' is not supervised"),
6022 )?]);
6023 };
6024
6025 self.route_outages.mark_operator_action(&module_id);
6026 if let Err(err) = module
6027 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6028 .await
6029 {
6030 self.route_outages
6031 .operator_action_ended_unrefused(&module_id);
6032 use crate::supervise::SuperviseError;
6033 let message = err.to_string();
6034 let error = match err {
6035 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6036 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6037 code: "swap_refused".to_string(),
6038 message,
6039 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6040 },
6041 SuperviseError::SwapFailed {
6042 arm,
6043 candidate_exit,
6044 ..
6045 } => ErrorBody {
6046 code: "swap_failed".to_string(),
6047 message,
6048 detail: Some(serde_json::json!({
6049 "arm": arm.as_str(),
6050 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6051 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6052 })),
6053 },
6054 _ => ErrorBody::new(
6055 "target_unavailable",
6056 format!("failed to swap module_id '{module_id}': {message}"),
6057 ),
6058 };
6059 return Ok(vec![control_error_body_frame(&frame, error)?]);
6060 }
6061 self.route_outages
6065 .operator_action_ended_unrefused(&module_id);
6066
6067 let response = ClientControlResponse::SupervisorAck {
6068 module_id,
6069 applied: true,
6070 };
6071 Ok(vec![control_response_body_frame(
6072 &frame,
6073 &response,
6074 "ClientControlResponse::SupervisorAck",
6075 )?])
6076 }
6077
6078 async fn handle_supervisor_reload(
6079 &self,
6080 frame: Frame,
6081 module_id: String,
6082 requester: Principal,
6083 ) -> Result<Vec<Frame>, RouterError> {
6084 let operation_lock = self.supervisor.operation_lock();
6085 let _operation_guard = operation_lock.lock().await;
6086 let Some(module) = self.supervisor.get(&module_id) else {
6087 return Ok(vec![control_error_frame(
6088 &frame,
6089 "unknown_module",
6090 format!("module_id '{module_id}' is not supervised"),
6091 )?]);
6092 };
6093
6094 self.route_outages.mark_operator_action(&module_id);
6095 if let Err(err) = module.reload_requested(Some(requester)).await {
6096 self.route_outages
6097 .operator_action_ended_unrefused(&module_id);
6098 let (code, message) = match err {
6099 crate::supervise::SuperviseError::Disabled { .. } => {
6100 ("module_disabled", err.to_string())
6101 }
6102 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6103 ("swap_in_progress", err.to_string())
6104 }
6105 _ => (
6106 "reload_failed",
6107 format!("failed to reload module_id '{module_id}': {err}"),
6108 ),
6109 };
6110 return Ok(vec![control_error_frame(&frame, code, message)?]);
6111 }
6112
6113 let response = ClientControlResponse::SupervisorAck {
6114 module_id,
6115 applied: true,
6116 };
6117 Ok(vec![control_response_body_frame(
6118 &frame,
6119 &response,
6120 "ClientControlResponse::SupervisorAck",
6121 )?])
6122 }
6123
6124 async fn handle_supervisor_rescan(
6128 &self,
6129 frame: Frame,
6130 mode: RescanMode,
6131 requester: Principal,
6132 ) -> Result<Vec<Frame>, RouterError> {
6133 let preview = mode.previews();
6134 let Some(context) = self.rescan.clone() else {
6135 return Ok(vec![control_error_frame(
6136 &frame,
6137 "rescan_unavailable",
6138 "the daemon was not started with a reloadable config path".to_string(),
6139 )?]);
6140 };
6141
6142 let operation_lock = self.supervisor.operation_lock();
6143 let _operation_guard = operation_lock.lock().await;
6144 let loaded = match self.load_rescan_config(&mode, &context) {
6148 Ok(config) => config,
6149 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6150 };
6151 let Some((config, activity_config)) = loaded else {
6159 return Ok(vec![control_error_frame(
6160 &frame,
6161 "invalid_daemon_config",
6162 format!(
6163 "daemon config not found at {}; refusing to rescan (an absent config would \
6164 retire every supervised module)",
6165 context.config_path.display()
6166 ),
6167 )?]);
6168 };
6169 let (
6170 configured_port,
6171 storage_config,
6172 admission_facts_carrier_module_id,
6173 admission_facts_targets,
6174 scope_authority_owners,
6175 privileged_operator_prompt,
6176 modules,
6177 reserved_capabilities,
6178 ) = (
6179 config.port,
6180 config.storage,
6181 config.admission_facts_carrier_module_id,
6182 config.admission_facts_targets,
6183 config.scope_authority_owners,
6184 config.privileged_operator_prompt,
6185 config.modules,
6186 config.reserved_capabilities,
6187 );
6188
6189 let mut restart_required = Vec::new();
6198 for section in RestartRequiredSection::ALL {
6199 let changed = match section {
6200 RestartRequiredSection::Port => configured_port != context.configured_port,
6201 RestartRequiredSection::Storage => storage_config != context.storage_config,
6202 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6203 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6204 }
6205 RestartRequiredSection::AdmissionFactsTargets => {
6206 admission_facts_targets != context.admission_facts_targets
6207 }
6208 RestartRequiredSection::ScopeAuthorityOwners => {
6209 scope_authority_owners != context.scope_authority_owners
6210 }
6211 RestartRequiredSection::PrivilegedOperatorPrompt => {
6212 privileged_operator_prompt != context.privileged_operator_prompt
6213 }
6214 };
6215 if changed {
6216 restart_required.push(section.label().to_string());
6217 }
6218 }
6219 if !restart_required.is_empty() {
6220 warn!(
6221 config_path = %context.config_path.display(),
6222 sections = %restart_required.join(", "),
6223 "daemon config changed outside the modules section; restart the daemon to apply those changes"
6224 );
6225 }
6226
6227 for configured in &modules {
6228 if let Err(err) = validate_spec(&configured.module_spec()) {
6229 return Ok(vec![control_error_frame(
6230 &frame,
6231 "invalid_daemon_config",
6232 format!("supervisor rescan rejected daemon config: {err}"),
6233 )?]);
6234 }
6235 }
6236
6237 let configured_capabilities = modules
6238 .iter()
6239 .map(|module| (module.module_id.clone(), module.enabled))
6240 .collect::<Vec<_>>();
6241 let preview_capability_warnings = if preview {
6242 let (_, registrations) = self.runtime_capability_snapshot()?;
6243 let current_modules = self
6244 .supervisor
6245 .list()
6246 .into_iter()
6247 .map(|module| module.module_id().to_string())
6248 .collect::<BTreeSet<_>>();
6249 let resulting_modules = configured_capabilities.clone();
6250 let removed = current_modules
6251 .into_iter()
6252 .filter(|module_id| {
6253 !resulting_modules
6254 .iter()
6255 .any(|(configured_id, _)| configured_id == module_id)
6256 })
6257 .collect::<Vec<_>>();
6258 let disabled = match &mode {
6261 RescanMode::Candidate(_) => resulting_modules
6262 .iter()
6263 .filter(|(_, enabled)| !enabled)
6264 .map(|(module_id, _)| module_id.clone())
6265 .collect::<Vec<_>>(),
6266 _ => Vec::new(),
6267 };
6268 self.capability_evaluator.preview_warnings(
6269 resulting_modules,
6270 &removed,
6271 &disabled,
6272 ®istrations,
6273 )
6274 } else {
6275 Vec::new()
6276 };
6277 let reconciled = match &mode {
6278 RescanMode::Candidate(candidate) => self
6279 .preview_candidate(&context, modules, candidate)
6280 .map(|result| (result, Vec::new())),
6281 _ => {
6282 self.reconcile_supervised_modules(
6283 &context.supervisor,
6284 modules,
6285 preview,
6286 requester.clone(),
6287 )
6288 .await
6289 }
6290 };
6291 let (result, changed) = match reconciled {
6292 Ok(result) => result,
6293 Err(message) => {
6294 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6295 }
6296 };
6297 if !preview {
6298 if let Some(ledger) = &self.activity {
6299 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6300 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6301 }
6302 if let Err(code) = self.configure_activity(activity_config).await {
6303 warn!(
6304 code,
6305 "activity retention could not be applied during rescan"
6306 );
6307 }
6308 self.capability_evaluator
6309 .configure(configured_capabilities, reserved_capabilities);
6310 self.capability_evaluator.wake_deadline_loop();
6311 self.refresh_capability_requirements();
6312 }
6313 let mut result = result;
6314 result.restart_required = restart_required;
6315 result.capability_warnings = preview_capability_warnings;
6316 let response = ClientControlResponse::SupervisorRescan { result };
6317 Ok(vec![control_response_body_frame(
6318 &frame,
6319 &response,
6320 "ClientControlResponse::SupervisorRescan",
6321 )?])
6322 }
6323
6324 async fn handle_supervisor_release_reserved(
6325 &self,
6326 frame: Frame,
6327 module_id: String,
6328 ) -> Result<Vec<Frame>, RouterError> {
6329 let Some(context) = self.rescan.clone() else {
6330 return Ok(vec![control_error_frame(
6331 &frame,
6332 "release_unavailable",
6333 "reserved-id release requires a daemon started with a reloadable config path",
6334 )?]);
6335 };
6336 let operation_lock = self.supervisor.operation_lock();
6337 let _operation_guard = operation_lock.lock().await;
6338 let loaded = match crate::daemon_config::load(&context.config_path) {
6339 Ok(Some(config)) => config,
6340 Ok(None) => {
6341 return Ok(vec![control_error_frame(
6342 &frame,
6343 "invalid_daemon_config",
6344 format!(
6345 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6346 context.config_path.display()
6347 ),
6348 )?])
6349 }
6350 Err(err) => {
6351 return Ok(vec![control_error_frame(
6352 &frame,
6353 "invalid_daemon_config",
6354 format!("unable to verify reserved-id release against daemon config: {err}"),
6355 )?])
6356 }
6357 };
6358 if loaded
6359 .modules
6360 .iter()
6361 .any(|configured| configured.module_id == module_id)
6362 {
6363 return Ok(vec![control_error_frame(
6364 &frame,
6365 "reserved_module_configured",
6366 format!(
6367 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6368 ),
6369 )?]);
6370 }
6371 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6372 return Ok(vec![control_error_frame(
6373 &frame,
6374 "reserved_gate_not_retained",
6375 format!(
6376 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6377 ),
6378 )?]);
6379 }
6380
6381 let response = ClientControlResponse::SupervisorAck {
6382 module_id,
6383 applied: true,
6384 };
6385 Ok(vec![control_response_body_frame(
6386 &frame,
6387 &response,
6388 "ClientControlResponse::SupervisorAck",
6389 )?])
6390 }
6391
6392 fn plan_rescan(
6396 &self,
6397 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6398 ) -> Result<RescanPlan, String> {
6399 let mut current = BTreeMap::new();
6400 for module in self.supervisor.list() {
6401 let (spec, health) = module.configuration().map_err(|err| {
6402 format!(
6403 "failed to read configuration for module_id '{}': {err}",
6404 module.module_id()
6405 )
6406 })?;
6407 let enabled = module
6408 .status()
6409 .map_err(|err| {
6410 format!(
6411 "failed to read status for module_id '{}': {err}",
6412 module.module_id()
6413 )
6414 })?
6415 .enabled;
6416 current.insert(
6417 module.module_id().to_string(),
6418 (module, spec, health, enabled),
6419 );
6420 }
6421 let configured = configured_modules
6422 .into_iter()
6423 .map(|module| (module.module_id.clone(), module))
6424 .collect::<BTreeMap<_, _>>();
6425
6426 let added = configured
6427 .keys()
6428 .filter(|module_id| !current.contains_key(*module_id))
6429 .cloned()
6430 .collect::<Vec<_>>();
6431 let removed = current
6432 .keys()
6433 .filter(|module_id| !configured.contains_key(*module_id))
6434 .cloned()
6435 .collect::<Vec<_>>();
6436 let mut changed_pending_reload = Vec::new();
6437 let mut configuration_changes = BTreeSet::new();
6438 let mut enabled_changes = BTreeSet::new();
6439 let mut unchanged = 0_u32;
6440
6441 for (module_id, configured_module) in &configured {
6442 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6443 else {
6444 continue;
6445 };
6446 let launch_changed = *current_spec != configured_module.module_spec();
6450 let configuration_changed =
6451 launch_changed || *current_health != configured_module.health;
6452 let enabled_changed = *current_enabled != configured_module.enabled;
6453 if configuration_changed {
6454 configuration_changes.insert(module_id.clone());
6455 }
6456 if launch_changed {
6457 changed_pending_reload.push(module_id.clone());
6458 }
6459 if enabled_changed {
6460 enabled_changes.insert(module_id.clone());
6461 }
6462 if !configuration_changed && !enabled_changed {
6463 unchanged = unchanged.saturating_add(1);
6464 }
6465 }
6466 Ok(RescanPlan {
6467 current,
6468 configured,
6469 added,
6470 removed,
6471 changed_pending_reload,
6472 configuration_changes,
6473 enabled_changes,
6474 unchanged,
6475 })
6476 }
6477
6478 async fn reconcile_supervised_modules(
6487 &self,
6488 supervisor: &Supervisor,
6489 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6490 preview: bool,
6491 requester: Principal,
6492 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
6493 let plan = self.plan_rescan(configured_modules)?;
6494
6495 if preview {
6501 return Ok((plan.preview_result(), Vec::new()));
6502 }
6503 let changed = plan
6504 .configuration_changes
6505 .union(&plan.enabled_changes)
6506 .cloned()
6507 .collect::<Vec<_>>();
6508 let RescanPlan {
6509 current,
6510 configured,
6511 added,
6512 removed,
6513 changed_pending_reload,
6514 configuration_changes,
6515 enabled_changes,
6516 unchanged,
6517 } = plan;
6518 if !(added.is_empty()
6523 && removed.is_empty()
6524 && configuration_changes.is_empty()
6525 && enabled_changes.is_empty())
6526 {
6527 if let Some(context) = &self.rescan {
6528 context
6529 .generation
6530 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
6531 }
6532 }
6533
6534 for module_id in &removed {
6535 let module = ¤t
6536 .get(module_id)
6537 .expect("removed module came from current supervisor state")
6538 .0;
6539 module
6540 .retire_requested(Some(requester.clone()))
6541 .await
6542 .map_err(|err| {
6543 format!("failed to retire module_id '{module_id}' during rescan: {err}")
6544 })?;
6545 self.supervisor.record_rescan_removal(module_id);
6579 self.supervisor.retire(module_id);
6580 self.route_outages.forget(module_id);
6581 }
6582
6583 for module_id in configured.keys() {
6584 let Some((module, _, _, _)) = current.get(module_id) else {
6585 continue;
6586 };
6587 let configured_module = configured
6588 .get(module_id)
6589 .expect("configured module id came from configured map");
6590 if configuration_changes.contains(module_id) {
6591 module
6592 .update_configuration(
6593 configured_module.module_spec(),
6594 configured_module.health.clone(),
6595 configured_module.drain_timeout_ms,
6596 )
6597 .await
6598 .map_err(|err| {
6599 format!(
6600 "failed to update module_id '{module_id}' configuration during rescan: {err}"
6601 )
6602 })?;
6603 }
6604 if enabled_changes.contains(module_id) {
6605 self.route_outages.mark_operator_action(module_id);
6608 module
6609 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
6610 .await
6611 .map_err(|err| {
6612 self.route_outages.operator_action_ended_unrefused(module_id);
6613 format!(
6614 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
6615 configured_module.enabled
6616 )
6617 })?;
6618 }
6619 }
6620
6621 for module_id in &added {
6622 let configured_module = configured
6623 .get(module_id)
6624 .expect("added module id came from configured map");
6625 supervisor
6626 .supervise_configured_requested(
6627 configured_module.module_spec(),
6628 configured_module.enabled,
6629 configured_module.health.clone(),
6630 configured_module.drain_timeout_ms,
6631 configured_module.restart,
6632 configured_module.register_timeout,
6633 Some(requester.clone()),
6634 )
6635 .map_err(|err| {
6636 format!("failed to add module_id '{module_id}' during rescan: {err}")
6637 })?;
6638 }
6639
6640 Ok((
6641 SupervisorRescanResult {
6642 added,
6643 removed,
6644 changed_pending_reload,
6645 enabled_changes: enabled_changes.iter().cloned().collect(),
6646 unchanged,
6647 preview: false,
6648 restart_required: Vec::new(),
6651 capability_warnings: Vec::new(),
6652 module_actions: Vec::new(),
6653 earlier_pending: Vec::new(),
6654 expect: None,
6655 },
6656 changed,
6657 ))
6658 }
6659
6660 async fn handle_supervisor_set_enabled(
6661 &self,
6662 frame: Frame,
6663 module_id: String,
6664 enabled: bool,
6665 requester: Principal,
6666 ) -> Result<Vec<Frame>, RouterError> {
6667 let operation_lock = self.supervisor.operation_lock();
6668 let _operation_guard = operation_lock.lock().await;
6669 let Some(module) = self.supervisor.get(&module_id) else {
6670 return Ok(vec![control_error_frame(
6671 &frame,
6672 "unknown_module",
6673 format!("module_id '{module_id}' is not supervised"),
6674 )?]);
6675 };
6676
6677 self.route_outages.mark_operator_action(&module_id);
6680 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
6681 Ok(applied) => applied,
6682 Err(err) => {
6683 self.route_outages
6684 .operator_action_ended_unrefused(&module_id);
6685 return Ok(vec![control_error_frame(
6686 &frame,
6687 "target_unavailable",
6688 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
6689 )?]);
6690 }
6691 };
6692 if !applied {
6693 self.route_outages
6696 .operator_action_ended_unrefused(&module_id);
6697 }
6698
6699 self.capability_evaluator.wake_deadline_loop();
6700 self.refresh_capability_requirements();
6701 let response = ClientControlResponse::SupervisorAck { module_id, applied };
6702 Ok(vec![control_response_body_frame(
6703 &frame,
6704 &response,
6705 "ClientControlResponse::SupervisorAck",
6706 )?])
6707 }
6708
6709 async fn handle_supervisor_health_probe(
6710 &self,
6711 frame: Frame,
6712 module_id: String,
6713 ) -> Result<Vec<Frame>, RouterError> {
6714 self.refresh_capability_requirements();
6715 let Some(registration) = self
6716 .registry
6717 .get_module(&module_id)
6718 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6719 else {
6720 return Ok(vec![control_error_frame(
6721 &frame,
6722 "unknown_module",
6723 format!("module_id '{module_id}' is not registered"),
6724 )?]);
6725 };
6726
6727 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
6740 {
6741 return Ok(vec![control_error_frame(
6742 &frame,
6743 "health_not_advertised",
6744 format!("module_id '{module_id}' did not advertise health.check"),
6745 )?]);
6746 }
6747
6748 let deadline = Instant::now() + self.health_probe_timeout;
6749 let pending = match self.forwarding.begin_module_control_rpc_for(
6750 &module_id,
6751 MODULE_CONTROL_OP_HEALTH_CHECK,
6752 deadline,
6753 ) {
6754 Ok(pending) => pending,
6755 Err(err) => {
6756 return Ok(vec![control_error_frame(
6757 &frame,
6758 forwarding_error_code(&err),
6759 err.to_string(),
6760 )?])
6761 }
6762 };
6763
6764 let PendingModuleControlRpc {
6765 endpoint,
6766 module_sink,
6767 negotiated_ver,
6768 corr: probe_corr,
6769 receiver,
6770 } = pending;
6771 let mut guard =
6772 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
6773 let probe_body =
6774 serde_json::to_vec(&ModuleControlRequest::HealthCheck {}).map_err(|err| {
6775 RouterError::backend(
6776 0,
6777 frame.header.corr,
6778 format!("failed to encode health.check request: {err}"),
6779 )
6780 })?;
6781 let probe_frame = Frame::build_with_version(
6782 negotiated_ver,
6783 FrameType::Request,
6784 control_flags(),
6785 0,
6786 0,
6787 probe_corr,
6788 probe_body,
6789 )
6790 .map_err(RouterError::FrameBuild)?;
6791
6792 match timeout_at(deadline, module_sink.send(probe_frame)).await {
6795 Ok(Ok(())) => {}
6796 Ok(Err(err)) => {
6797 return Ok(vec![control_error_frame(
6798 &frame,
6799 "target_unavailable",
6800 err.to_string(),
6801 )?]);
6802 }
6803 Err(_) => {
6804 return Ok(vec![control_error_frame(
6805 &frame,
6806 "module_timeout",
6807 format!(
6808 "module_id '{module_id}' did not answer health.check within {:?}",
6809 self.health_probe_timeout
6810 ),
6811 )?]);
6812 }
6813 }
6814
6815 match timeout_at(deadline, receiver).await {
6816 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
6817 guard.disarm();
6818 let Some(report) = response.health_report() else {
6819 return Ok(vec![control_error_frame(
6820 &frame,
6821 "invalid_control_body",
6822 "health.check RPC returned a non-health response",
6823 )?]);
6824 };
6825 let HealthReport {
6830 status,
6831 detail,
6832 metrics,
6833 } = report;
6834 let capability_detail = self
6835 .capability_evaluator
6836 .required_problem_detail(&module_id);
6837 let response = ClientControlResponse::SupervisorHealthProbe {
6838 module_id,
6839 status,
6840 detail: append_capability_problem_detail(detail, capability_detail),
6841 metrics,
6842 };
6843 Ok(vec![control_response_body_frame(
6844 &frame,
6845 &response,
6846 "ClientControlResponse::SupervisorHealthProbe",
6847 )?])
6848 }
6849 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
6850 guard.disarm();
6851 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
6854 }
6855 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
6856 guard.disarm();
6857 Ok(vec![control_error_frame(
6858 &frame,
6859 "target_unavailable",
6860 message,
6861 )?])
6862 }
6863 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
6864 guard.disarm();
6865 Ok(vec![control_error_frame(
6866 &frame,
6867 "invalid_control_body",
6868 message,
6869 )?])
6870 }
6871 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
6872 guard.disarm();
6873 Ok(vec![control_error_frame(
6874 &frame,
6875 "invalid_control_body",
6876 format!("expected module-control op '{expected}', got '{actual}'"),
6877 )?])
6878 }
6879 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
6880 guard.disarm();
6881 Ok(vec![control_error_frame(
6882 &frame,
6883 "module_timeout",
6884 format!(
6885 "module_id '{module_id}' answered health.check after {:?}",
6886 self.health_probe_timeout
6887 ),
6888 )?])
6889 }
6890 Ok(Err(_)) => Ok(vec![control_error_frame(
6891 &frame,
6892 "target_unavailable",
6893 "health.check waiter was canceled before the module responded",
6894 )?]),
6895 Err(_) => Ok(vec![control_error_frame(
6896 &frame,
6897 "module_timeout",
6898 format!(
6899 "module_id '{module_id}' did not answer health.check within {:?}",
6900 self.health_probe_timeout
6901 ),
6902 )?]),
6903 }
6904 }
6905
6906 fn supervisor_status(
6907 &self,
6908 module_id: &str,
6909 corr: u64,
6910 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
6911 self.supervisor
6912 .get(module_id)
6913 .map(|module| {
6914 let warming = module.is_warming_for_control("status").map_err(|err| {
6915 RouterError::backend(
6916 0,
6917 corr,
6918 format!(
6919 "failed to read supervisor warming state for module_id '{module_id}': {err}"
6920 ),
6921 )
6922 })?;
6923 module.status_for_control("status").map_err(|err| {
6924 RouterError::backend(
6925 0,
6926 corr,
6927 format!(
6928 "failed to read supervisor status for module_id '{module_id}': {err}"
6929 ),
6930 )
6931 }).map(|status| (status, warming))
6932 })
6933 .transpose()
6934 }
6935
6936 fn guard_module_control_op(
6937 &self,
6938 frame: &Frame,
6939 module_id: &str,
6940 op: &str,
6941 ) -> Result<Option<Frame>, RouterError> {
6942 if self.module_grants_op(module_id, op, frame.header.corr)? {
6943 return Ok(None);
6944 }
6945
6946 Ok(Some(control_error_frame(
6947 frame,
6948 "op_not_allowed",
6949 format!("module_id '{module_id}' did not grant control op '{op}'"),
6950 )?))
6951 }
6952
6953 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
6954 let Some(registration) = self
6955 .registry
6956 .get_module(module_id)
6957 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
6958 else {
6959 return Ok(false);
6960 };
6961 Ok(module_registration_grants_op(®istration.control_ops, op))
6962 }
6963
6964 fn handle_status_update(
6965 &self,
6966 endpoint: ModuleEndpointId,
6967 frame: Frame,
6968 ) -> Result<Vec<Frame>, RouterError> {
6969 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
6970 Ok(update) => update,
6971 Err(err) => {
6972 if is_known_module_push_op(&frame.body) {
6977 return Ok(vec![control_error_frame(
6978 &frame,
6979 "invalid_control_body",
6980 format!("malformed module control push body: {err}"),
6981 )?]);
6982 }
6983 return Ok(Vec::new());
6984 }
6985 };
6986
6987 match update {
6988 ModuleControlPush::RouteStatus {
6989 route_channel,
6990 route_epoch,
6991 status,
6992 } => {
6993 self.forwarding
6994 .cache_status(endpoint, route_channel, route_epoch, status)
6995 .map_err(RouterError::Forwarding)?;
6996 }
6997 }
6998 Ok(Vec::new())
6999 }
7000
7001 fn handle_route_poll(
7002 &self,
7003 ctx: &RouteCtx,
7004 frame: Frame,
7005 route_channel: u16,
7006 route_epoch: u32,
7007 kind: PollKind,
7008 ) -> Result<Vec<Frame>, RouterError> {
7009 let snapshot = self
7010 .forwarding
7011 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7012 .map_err(RouterError::Forwarding)?;
7013 let response = match (kind, snapshot) {
7014 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7015 ClientControlResponse::RoutePoll {
7016 route_channel,
7017 route_epoch,
7018 status,
7019 live: None,
7020 }
7021 }
7022 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7023 route_channel,
7024 route_epoch,
7025 status: None,
7026 live: None,
7027 },
7028 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7029 let live = self
7055 .process_liveness
7056 .as_ref()
7057 .and_then(|source| source.process_live(&module_id))
7058 .unwrap_or(true);
7059 ClientControlResponse::RoutePoll {
7060 route_channel,
7061 route_epoch,
7062 status: None,
7063 live: Some(live),
7064 }
7065 }
7066 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7067 route_channel,
7068 route_epoch,
7069 status: None,
7070 live: Some(false),
7071 },
7072 };
7073
7074 Ok(vec![control_response_body_frame(
7075 &frame,
7076 &response,
7077 "ClientControlResponse::RoutePoll",
7078 )?])
7079 }
7080
7081 pub(crate) fn observe_module_control_completion(
7082 &self,
7083 completion: ModuleControlRpcCompletion,
7084 ) -> bool {
7085 match completion {
7086 ModuleControlRpcCompletion::Unknown => false,
7087 ModuleControlRpcCompletion::Settled => true,
7088 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7089 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7090 info!(
7091 module_id = %module_id,
7092 latency_ms,
7093 "late health.check answer proves the module is alive"
7094 );
7095 match self
7096 .supervisor
7097 .record_late_health_answer(&module_id, latency_ms)
7098 {
7099 Ok(true) => {}
7100 Ok(false) => debug!(
7101 module_id = %module_id,
7102 latency_ms,
7103 "late health.check answer has no active supervisor snapshot"
7104 ),
7105 Err(err) => warn!(
7106 module_id = %module_id,
7107 latency_ms,
7108 error = %err,
7109 "failed to record late health.check answer"
7110 ),
7111 }
7112 true
7113 }
7114 }
7115 }
7116
7117 fn refuse_to_end_module_connection_for_a_client(
7138 &self,
7139 module_connection_id: ConnectionId,
7140 corr: u64,
7141 err: ForwardingError,
7142 ) -> Result<(), RouterError> {
7143 if let ForwardingError::ConnectionClosing { connection_id } = err {
7144 if connection_id != module_connection_id {
7145 warn!(
7146 module_connection_id = module_connection_id.get(),
7147 client_connection_id = connection_id.get(),
7148 corr,
7149 "dropping a route.bind response for a closing client; the module connection keeps serving"
7150 );
7151 return Ok(());
7152 }
7153 }
7154 Err(RouterError::Forwarding(err))
7155 }
7156
7157 fn handle_module_relay_response(
7158 &self,
7159 connection_id: ConnectionId,
7160 frame: Frame,
7161 ) -> Result<Vec<Frame>, RouterError> {
7162 let mut secondary_error = None;
7163 let outcome = match frame.header.ty {
7164 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7165 Ok(probe) if probe.op == "route.bind" => {
7166 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7167 Ok(ModuleControlResponse::RouteBindAck {}) => {
7168 RouteBindRelayOutcome::Accepted
7169 }
7170 Ok(other) => {
7171 let message =
7172 format!("route.bind response carried unexpected body: {other:?}");
7173 secondary_error = Some(control_error_frame(
7174 &frame,
7175 "invalid_control_body",
7176 message.clone(),
7177 )?);
7178 RouteBindRelayOutcome::ModuleGone(message)
7179 }
7180 Err(err) => {
7181 let message = format!("malformed route.bind response body: {err}");
7182 secondary_error = Some(control_error_frame(
7183 &frame,
7184 "invalid_control_body",
7185 message.clone(),
7186 )?);
7187 RouteBindRelayOutcome::ModuleGone(message)
7188 }
7189 }
7190 }
7191 Ok(probe) => {
7192 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7193 {
7194 Ok(response) => ModuleControlRpcOutcome::Response(response),
7195 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7196 "malformed {} response body: {err}",
7197 probe.op
7198 )),
7199 };
7200 let completion = self
7201 .forwarding
7202 .complete_module_control_rpc(
7203 connection_id,
7204 frame.header.corr,
7205 Some(&probe.op),
7206 outcome,
7207 )
7208 .map_err(RouterError::Forwarding)?;
7209 if !self.observe_module_control_completion(completion) {
7210 debug!(
7211 connection_id = connection_id.get(),
7212 corr = frame.header.corr,
7213 op = %probe.op,
7214 "dropping late or unknown module-control RPC response"
7215 );
7216 }
7217 return Ok(Vec::new());
7218 }
7219 Err(err) => {
7220 if let Some(expected_op) = self
7221 .forwarding
7222 .pending_module_control_op(connection_id, frame.header.corr)
7223 .map_err(RouterError::Forwarding)?
7224 {
7225 let completion = self
7226 .forwarding
7227 .complete_module_control_rpc(
7228 connection_id,
7229 frame.header.corr,
7230 None,
7231 ModuleControlRpcOutcome::MalformedResponse(format!(
7232 "malformed {expected_op} response body: {err}"
7233 )),
7234 )
7235 .map_err(RouterError::Forwarding)?;
7236 if !self.observe_module_control_completion(completion) {
7237 debug!(
7238 connection_id = connection_id.get(),
7239 corr = frame.header.corr,
7240 "dropping late malformed module-control RPC response"
7241 );
7242 }
7243 return Ok(Vec::new());
7244 }
7245 let message = format!("malformed route.bind response body: {err}");
7246 secondary_error = Some(control_error_frame(
7247 &frame,
7248 "invalid_control_body",
7249 message.clone(),
7250 )?);
7251 RouteBindRelayOutcome::ModuleGone(message)
7252 }
7253 },
7254 FrameType::Error => {
7255 if self
7256 .forwarding
7257 .pending_module_control_op(connection_id, frame.header.corr)
7258 .map_err(RouterError::Forwarding)?
7259 .is_some()
7260 {
7261 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7262 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7263 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7264 "malformed module-control ERROR body: {err}"
7265 )),
7266 };
7267 let completion = self
7268 .forwarding
7269 .complete_module_control_rpc(
7270 connection_id,
7271 frame.header.corr,
7272 None,
7273 outcome,
7274 )
7275 .map_err(RouterError::Forwarding)?;
7276 if !self.observe_module_control_completion(completion) {
7277 debug!(
7278 connection_id = connection_id.get(),
7279 corr = frame.header.corr,
7280 "dropping late or unknown module-control RPC error"
7281 );
7282 }
7283 return Ok(Vec::new());
7284 }
7285 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7286 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7287 Err(err) => {
7288 let message = format!("malformed route.bind ERROR body: {err}");
7289 secondary_error = Some(control_error_frame(
7290 &frame,
7291 "invalid_control_body",
7292 message.clone(),
7293 )?);
7294 RouteBindRelayOutcome::ModuleGone(message)
7295 }
7296 }
7297 }
7298 ty => {
7299 return Ok(vec![control_error_frame(
7300 &frame,
7301 "unsupported_control_frame",
7302 format!("unsupported module channel-0 frame {ty:?}"),
7303 )?])
7304 }
7305 };
7306
7307 let settled =
7308 self.forwarding
7309 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7310 let completion = match settled {
7311 Ok(completion) => completion,
7312 Err(err) => {
7313 self.refuse_to_end_module_connection_for_a_client(
7314 connection_id,
7315 frame.header.corr,
7316 err,
7317 )?;
7318 return Ok(secondary_error.into_iter().collect());
7319 }
7320 };
7321 if let Some(target) = completion.abandoned.as_ref() {
7322 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7323 }
7324 if !completion.settled {
7325 debug!(
7326 connection_id = connection_id.get(),
7327 corr = frame.header.corr,
7328 frame_type = ?frame.header.ty,
7329 "dropping late or unknown route.bind relay response"
7330 );
7331 }
7332 Ok(secondary_error.into_iter().collect())
7333 }
7334
7335 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7336 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7337 self.cleanup_connection_with_end_reason(
7341 connection_id,
7342 RegistrationEndReason::ExplicitGoodbye,
7343 )
7344 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7345 Ok(Vec::new())
7346 }
7347}
7348
7349impl Default for ControlHandler {
7350 fn default() -> Self {
7351 Self::new(Arc::new(Registry::default()))
7352 }
7353}
7354
7355impl crate::supervise::SwapPromotionObserver for ControlHandler {
7356 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7357 self.apply_registration_capabilities(registration);
7358 }
7359}
7360
7361fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7362 CapabilityRequirementStatus {
7363 consumer: status.consumer,
7364 capability: status.capability,
7365 need: match status.need {
7366 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7367 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7368 },
7369 verdict: status.verdict.as_str().to_string(),
7370 episode_seq: status.episode_seq,
7371 config_satisfiable: status.config_satisfiable,
7372 runtime_available: status.runtime_available,
7373 detail: status.detail,
7374 }
7375}
7376
7377fn append_capability_problem_detail(
7378 detail: Option<String>,
7379 capability_detail: Option<String>,
7380) -> Option<String> {
7381 match (detail, capability_detail) {
7382 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7383 (Some(detail), None) => Some(detail),
7384 (None, Some(capability_detail)) => Some(capability_detail),
7385 (None, None) => None,
7386 }
7387}
7388
7389fn subc_ops() -> Vec<String> {
7390 SUBC_CONTROL_OPS
7391 .iter()
7392 .map(|op| (*op).to_string())
7393 .collect()
7394}
7395
7396fn module_subc_ops() -> Vec<String> {
7397 SUBC_CONTROL_OPS
7398 .iter()
7399 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7400 .map(|op| (*op).to_string())
7401 .collect()
7402}
7403
7404#[cfg(test)]
7405fn module_baseline_control_ops() -> Vec<String> {
7406 MODULE_BASELINE_CONTROL_OPS
7407 .iter()
7408 .map(|op| (*op).to_string())
7409 .collect()
7410}
7411
7412fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7413 let mut seen = HashSet::new();
7414 let mut effective = Vec::new();
7415 for op in MODULE_BASELINE_CONTROL_OPS {
7416 if seen.insert((*op).to_string()) {
7417 effective.push((*op).to_string());
7418 }
7419 }
7420 for op in declared.unwrap_or_default() {
7421 if seen.insert(op.clone()) {
7422 effective.push(op);
7423 }
7424 }
7425 effective
7426}
7427
7428fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
7429 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
7430}
7431
7432fn target_module_id(target: &RouteTarget) -> &str {
7433 match target {
7434 RouteTarget::ToolProvider { module_id }
7435 | RouteTarget::ManagementSurface { module_id }
7436 | RouteTarget::InternalService { module_id, .. } => module_id,
7437 }
7438}
7439
7440fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
7441 roles.iter().any(|role| match (target, role) {
7442 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
7443 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
7444 (
7445 RouteTarget::InternalService { service_id, .. },
7446 ProviderRole::InternalService {
7447 service_id: provided,
7448 ..
7449 },
7450 ) => service_id == provided,
7451 _ => false,
7452 })
7453}
7454
7455fn is_routable_role(role: &ProviderRole) -> bool {
7456 matches!(
7457 role,
7458 ProviderRole::ToolProvider { .. }
7459 | ProviderRole::ManagementSurface { .. }
7460 | ProviderRole::InternalService { .. }
7461 )
7462}
7463
7464#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7465enum ControlRequestBodyError {
7466 UnknownOp,
7467 InvalidBody,
7468}
7469
7470#[derive(Debug, Deserialize)]
7471struct ControlOpProbe {
7472 op: String,
7473}
7474
7475const MODULE_PUSH_OPS: &[&str] = &["route.status"];
7478
7479fn is_known_module_push_op(body: &[u8]) -> bool {
7480 serde_json::from_slice::<ControlOpProbe>(body)
7481 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
7482 .unwrap_or(false)
7483}
7484
7485fn is_known_module_request_op(body: &[u8]) -> bool {
7486 serde_json::from_slice::<ControlOpProbe>(body)
7487 .map(|probe| is_module_to_subc_op(&probe.op))
7488 .unwrap_or(false)
7489}
7490
7491fn is_module_to_subc_op(op: &str) -> bool {
7492 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
7493}
7494
7495fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
7496 debug!(
7497 op = %op,
7498 connection_id = connection_id.get(),
7499 corr,
7500 "control dispatch"
7501 );
7502}
7503
7504fn log_slow_control_dispatch(
7505 dispatch_started_at: Option<StdInstant>,
7506 op: &'static str,
7507 connection_id: ConnectionId,
7508 corr: u64,
7509 route_open_timings: Option<&RouteOpenTimings>,
7510) {
7511 let Some(dispatch_started_at) = dispatch_started_at else {
7512 return;
7513 };
7514 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
7516 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
7517 if let Some(timings) = route_open_timings {
7518 warn!(
7519 op = %op,
7520 connection_id = connection_id.get(),
7521 corr,
7522 elapsed_ms = elapsed.as_millis() as u64,
7523 module_id = ?timings.module_id,
7524 principal = %timings.principal,
7525 admission_ms = timings.admission.as_millis() as u64,
7526 project_root_ms = timings.project_root.as_millis() as u64,
7527 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
7528 bind_send_ms = timings.bind_send.as_millis() as u64,
7529 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
7530 "slow control dispatch"
7531 );
7532 return;
7533 }
7534 warn!(
7535 op = %op,
7536 connection_id = connection_id.get(),
7537 corr,
7538 elapsed_ms = elapsed.as_millis() as u64,
7539 "slow control dispatch"
7540 );
7541 }
7542}
7543
7544#[cfg(test)]
7546type TestCallerLookup =
7547 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
7548
7549fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
7551 match request {
7552 ClientControlRequest::SupervisorRestart { module_id, .. } => {
7553 Some(PrivilegedOperation::Restart { module_id })
7554 }
7555 ClientControlRequest::SupervisorSwap { module_id, .. } => {
7556 Some(PrivilegedOperation::Swap { module_id })
7557 }
7558 ClientControlRequest::SupervisorReload { module_id } => {
7559 Some(PrivilegedOperation::Reload { module_id })
7560 }
7561 ClientControlRequest::SupervisorRescan { preview } => {
7562 if *preview {
7563 None
7564 } else {
7565 Some(PrivilegedOperation::Rescan)
7566 }
7567 }
7568 ClientControlRequest::SupervisorRescanChecked {
7573 preview,
7574 candidate,
7575 expect,
7576 } => (!*preview && candidate.is_none() && expect.is_some())
7577 .then_some(PrivilegedOperation::Rescan),
7578 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
7579 Some(PrivilegedOperation::SetEnabled {
7580 module_id,
7581 enabled: *enabled,
7582 })
7583 }
7584 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
7585 Some(PrivilegedOperation::ReleaseReserved { module_id })
7586 }
7587 ClientControlRequest::ServerDescribe {}
7588 | ClientControlRequest::CatalogList { .. }
7589 | ClientControlRequest::RouteOpen { .. }
7590 | ClientControlRequest::RoutePoll { .. }
7591 | ClientControlRequest::SupervisorList {}
7592 | ClientControlRequest::SupervisorHealth {}
7593 | ClientControlRequest::SupervisorHealthProbe { .. }
7594 | ClientControlRequest::SupervisorRoutes { .. }
7595 | ClientControlRequest::SupervisorProvenance { .. }
7596 | ClientControlRequest::SupervisorStderrTail { .. }
7597 | ClientControlRequest::SupervisorTerminals { .. }
7598 | ClientControlRequest::SupervisorSpawnSnapshot {}
7599 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
7600 }
7601}
7602
7603fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
7604 match request {
7605 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
7606 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
7607 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
7608 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
7609 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
7610 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
7611 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
7612 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
7613 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
7614 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
7615 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
7616 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
7617 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
7618 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
7619 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
7620 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
7621 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
7622 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
7623 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
7624 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
7625 }
7626}
7627
7628struct RouteOpenCaller {
7636 principal: String,
7640 scope: String,
7643 harness: String,
7646}
7647
7648impl RouteOpenCaller {
7649 fn new(
7650 principal: Option<&Principal>,
7651 scope: Option<&ScopeSelector>,
7652 identity: &BindIdentity,
7653 ) -> Self {
7654 Self {
7655 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
7656 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
7657 harness: bounded_log_text(&identity.harness),
7658 }
7659 }
7660
7661 fn unknown() -> Self {
7664 Self {
7665 principal: "unknown".to_string(),
7666 scope: "unknown".to_string(),
7667 harness: "unknown".to_string(),
7668 }
7669 }
7670}
7671
7672const MAX_LOGGED_TEXT_CHARS: usize = 64;
7675
7676fn bounded_log_text(value: &str) -> String {
7680 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
7681 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
7682 bounded.push('…');
7683 }
7684 format!("{bounded:?}")
7685}
7686
7687fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
7693 use subc_protocol::scope::ScopeKind;
7694 let kind = match stamp.kind {
7695 ScopeKind::Head => "head",
7696 ScopeKind::Worker => "worker",
7697 ScopeKind::Ephemeral => "ephemeral",
7698 };
7699 format!(
7700 "{kind}/{}/{}@{}",
7701 principal_label(&stamp.owner),
7702 bounded_log_text(&stamp.scope_ref),
7703 stamp.scope_epoch
7704 )
7705}
7706
7707fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
7714 let epoch = selector
7715 .scope_epoch
7716 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
7717 format!(
7718 "requested/{}/{}@{epoch}",
7719 bounded_log_text(&principal_label(&selector.owner)),
7720 bounded_log_text(&selector.scope_ref),
7721 )
7722}
7723
7724fn principal_label(principal: &Principal) -> String {
7725 match principal {
7726 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
7727 Principal::Direct => "direct".to_string(),
7728 other => format!("{other:?}"),
7729 }
7730}
7731
7732fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
7733 match request {
7734 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
7735 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
7736 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
7737 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
7738 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
7739 }
7740}
7741
7742fn parse_client_control_request(
7743 body: &[u8],
7744) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
7745 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
7746 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7747 Ok(probe)
7748 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
7749 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
7750 {
7751 ControlRequestBodyError::InvalidBody
7752 }
7753 Ok(_) => ControlRequestBodyError::UnknownOp,
7754 Err(_) => ControlRequestBodyError::InvalidBody,
7755 };
7756 (err, classification)
7757 })
7758}
7759
7760fn parse_module_control_request_from_module(
7761 body: &[u8],
7762) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
7763 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
7764 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
7765 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
7766 Ok(_) => ControlRequestBodyError::UnknownOp,
7767 Err(_) => ControlRequestBodyError::InvalidBody,
7768 };
7769 (err, classification)
7770 })
7771}
7772
7773#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
7774enum ProviderRoleKind {
7775 ToolProvider,
7776 PipelineStage,
7777 ManagementSurface,
7778 InternalService,
7779}
7780
7781fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
7782 match role {
7783 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
7784 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
7785 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
7786 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
7787 }
7788}
7789
7790fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
7791 roles.iter().map(provider_role_kind).collect()
7792}
7793
7794const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
7797
7798#[derive(Debug, Default, PartialEq, Eq)]
7800struct ScopeOutcomeCounts {
7801 created: usize,
7802 replaced: usize,
7803 updated: usize,
7804 unchanged: usize,
7805 refused: usize,
7806}
7807
7808impl ScopeOutcomeCounts {
7809 fn of(results: &[ScopeRecordResult]) -> Self {
7810 let mut counts = Self::default();
7811 for result in results {
7812 let slot = match result.outcome {
7813 ScopeRecordOutcome::Created => &mut counts.created,
7814 ScopeRecordOutcome::Replaced => &mut counts.replaced,
7815 ScopeRecordOutcome::Updated => &mut counts.updated,
7816 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
7817 ScopeRecordOutcome::Refused => &mut counts.refused,
7818 };
7819 *slot += 1;
7820 }
7821 counts
7822 }
7823}
7824
7825#[cfg(test)]
7826mod scope_outcome_count_tests {
7827 use super::*;
7828
7829 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
7830 ScopeRecordResult {
7831 scope_ref: "r".to_string(),
7832 scope_epoch: 1,
7833 outcome,
7834 code: None,
7835 message: None,
7836 version: None,
7837 parent_state: None,
7838 }
7839 }
7840
7841 #[test]
7844 fn every_outcome_is_counted_in_its_own_field() {
7845 let results = [
7846 result(ScopeRecordOutcome::Created),
7847 result(ScopeRecordOutcome::Created),
7848 result(ScopeRecordOutcome::Replaced),
7849 result(ScopeRecordOutcome::Updated),
7850 result(ScopeRecordOutcome::Unchanged),
7851 result(ScopeRecordOutcome::Refused),
7852 result(ScopeRecordOutcome::Refused),
7853 result(ScopeRecordOutcome::Refused),
7854 ];
7855 assert_eq!(
7856 ScopeOutcomeCounts::of(&results),
7857 ScopeOutcomeCounts {
7858 created: 2,
7859 replaced: 1,
7860 updated: 1,
7861 unchanged: 1,
7862 refused: 3,
7863 }
7864 );
7865 }
7866}
7867
7868fn capability_census_trigger(
7872 old: Option<&CapabilityDeclarations>,
7873 new: Option<&CapabilityDeclarations>,
7874) -> bool {
7875 let old_provides = old
7876 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
7877 .unwrap_or_default();
7878 let old_denies = old
7879 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
7880 .unwrap_or_default();
7881 let new = new.cloned().unwrap_or(CapabilityDeclarations {
7882 provides: Vec::new(),
7883 requires: Vec::new(),
7884 must_never_reach: Vec::new(),
7885 });
7886
7887 new.provides
7888 .iter()
7889 .any(|capability| !old_provides.contains(capability))
7890 || new
7891 .must_never_reach
7892 .iter()
7893 .any(|capability| !old_denies.contains(capability))
7894}
7895
7896fn denied_capability<'a>(
7899 opening_manifest: &'a ModuleManifest,
7900 target_manifest: &ModuleManifest,
7901) -> Option<&'a str> {
7902 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
7903 let target_capabilities = target_manifest.capabilities.as_ref()?;
7904 opening_capabilities
7905 .must_never_reach
7906 .iter()
7907 .find(|denied| {
7908 target_capabilities
7909 .provides
7910 .iter()
7911 .any(|provided| provided == *denied)
7912 })
7913 .map(String::as_str)
7914}
7915
7916fn catalog_update_frozen_field_message(
7917 registered: &ModuleManifest,
7918 provides: &[ProviderRole],
7919) -> Option<String> {
7920 let old_has_provides = !registered.provides.is_empty();
7921 let new_has_provides = !provides.is_empty();
7922 if old_has_provides != new_has_provides {
7923 return Some(format!(
7924 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
7925 registered.module_id
7926 ));
7927 }
7928
7929 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
7930 return Some(format!(
7931 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
7932 registered.module_id
7933 ));
7934 }
7935
7936 let registered_concurrency = manifest_concurrency(registered);
7937 let mut candidate = registered.clone();
7938 candidate.provides = provides.to_vec();
7939 let candidate_concurrency = manifest_concurrency(&candidate);
7940 if candidate_concurrency != registered_concurrency {
7941 return Some(format!(
7942 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
7943 registered.module_id, registered_concurrency, candidate_concurrency
7944 ));
7945 }
7946
7947 None
7950}
7951
7952fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
7953 manifest.provides.iter().any(is_routable_role)
7954}
7955
7956fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
7962 manifest
7963 .provides
7964 .iter()
7965 .find_map(|provider| match provider {
7966 ProviderRole::ToolProvider { concurrency, .. }
7967 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
7968 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
7969 })
7970 .unwrap_or(Concurrency::ModuleManaged)
7971}
7972
7973fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
7980 let has_management_surface = manifest
7981 .provides
7982 .iter()
7983 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
7984 if !has_management_surface {
7985 return false;
7986 }
7987 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
7988 return false;
7989 };
7990 let Some(provides) = raw
7991 .get("manifest")
7992 .and_then(|manifest| manifest.get("provides"))
7993 .and_then(serde_json::Value::as_array)
7994 else {
7995 return false;
7996 };
7997 provides.iter().any(|role| {
8002 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8003 && role.get("concurrency").is_none()
8004 })
8005}
8006
8007fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8008 if peer_version != PROTOCOL_VERSION {
8009 return Err(format!(
8010 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8011 ));
8012 }
8013 Ok(PROTOCOL_VERSION)
8014}
8015
8016fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8017 Frame::build_with_version(
8018 response_version(frame),
8019 FrameType::Pong,
8020 frame.header.flags,
8021 0,
8022 0,
8023 frame.header.corr,
8024 Vec::new(),
8025 )
8026 .map_err(RouterError::FrameBuild)
8027}
8028
8029pub(crate) fn control_error_frame(
8030 frame: &Frame,
8031 code: &'static str,
8032 message: impl Into<String>,
8033) -> Result<Frame, RouterError> {
8034 control_error_body_frame(
8035 frame,
8036 ErrorBody {
8037 code: code.to_string(),
8038 message: message.into(),
8039 detail: None,
8040 },
8041 )
8042}
8043
8044fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8052 control_error_frame_with_flags(frame, error, daemon_error_flags())
8053}
8054
8055fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8061 control_error_frame_with_flags(frame, error, control_flags())
8062}
8063
8064fn control_error_frame_with_flags(
8065 frame: &Frame,
8066 error: ErrorBody,
8067 flags: Flags,
8068) -> Result<Frame, RouterError> {
8069 let body = serde_json::to_vec(&error).map_err(|err| {
8070 RouterError::backend(
8071 0,
8072 frame.header.corr,
8073 format!("failed to encode control ERROR: {err}"),
8074 )
8075 })?;
8076
8077 Frame::build_with_version(
8078 response_version(frame),
8079 FrameType::Error,
8080 flags,
8081 0,
8082 0,
8083 frame.header.corr,
8084 body,
8085 )
8086 .map_err(RouterError::FrameBuild)
8087}
8088
8089fn control_response_body_frame<T: Serialize>(
8090 frame: &Frame,
8091 reply: &T,
8092 label: &'static str,
8093) -> Result<Frame, RouterError> {
8094 let body = serde_json::to_vec(reply).map_err(|err| {
8095 RouterError::backend(
8096 0,
8097 frame.header.corr,
8098 format!("failed to encode {label}: {err}"),
8099 )
8100 })?;
8101
8102 Frame::build_with_version(
8103 response_version(frame),
8104 FrameType::Response,
8105 control_flags(),
8106 0,
8107 0,
8108 frame.header.corr,
8109 body,
8110 )
8111 .map_err(RouterError::FrameBuild)
8112}
8113
8114fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8144 match err {
8145 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8146 ForwardingError::NoModuleConnection => "target_unavailable",
8147 ForwardingError::ModuleReloading { .. } => "module_reloading",
8148 ForwardingError::ClientRouteChannelExhausted { .. }
8149 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8150 ForwardingError::StaleModuleEndpoint
8151 | ForwardingError::UnknownReservation { .. }
8152 | ForwardingError::ConnectionClosing { .. }
8153 | ForwardingError::ClientEgressClosed { .. }
8154 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8155 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8158 ForwardingError::RelayCorrelationExhausted
8159 | ForwardingError::RouteOpenBuild(_)
8160 | ForwardingError::Poisoned => "forwarding_error",
8161 }
8162}
8163
8164fn response_version(frame: &Frame) -> u8 {
8165 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8166 frame.header.ver
8167 } else {
8168 PROTOCOL_VERSION
8169 }
8170}
8171
8172fn control_flags() -> Flags {
8173 Flags::new(false, Priority::Passive, false)
8174}
8175
8176fn send_goodbye_target_best_effort(
8181 counters: &DaemonCounters,
8182 target: &GoodbyeTarget,
8183 context: &'static str,
8184) {
8185 let Ok(frame) = Frame::build_with_version(
8186 target.negotiated_ver,
8187 FrameType::Goodbye,
8188 control_flags(),
8189 target.channel,
8190 target.epoch,
8191 0,
8192 Vec::new(),
8193 ) else {
8194 return;
8195 };
8196 crate::forwarding::send_module_route_goodbye(
8197 counters,
8198 &target.sink,
8199 frame,
8200 target.module_id.as_deref(),
8201 context,
8202 );
8203}
8204
8205pub(crate) fn send_route_control_pushes(
8206 forwarding: &ForwardingTable,
8207 routes: Vec<EndpointRoute>,
8208 push: ClientControlPush,
8209) {
8210 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8211 for route in routes {
8212 let target = route.goodbye_target;
8213 if let Some((existing, channels)) = targets
8214 .iter_mut()
8215 .find(|(existing, _)| existing.connection_id == target.connection_id)
8216 {
8217 debug_assert_eq!(
8218 existing.negotiated_ver, target.negotiated_ver,
8219 "one connection cannot negotiate multiple frame versions"
8220 );
8221 if !channels.contains(&target.channel) {
8222 channels.push(target.channel);
8223 }
8224 continue;
8225 }
8226 let channel = target.channel;
8227 targets.push((target, vec![channel]));
8228 }
8229 for (target, mut channels) in targets {
8230 channels.sort_unstable();
8231 let mut push = push.clone();
8232 match &mut push {
8233 ClientControlPush::RouteClosing {
8234 channels: covered, ..
8235 }
8236 | ClientControlPush::RouteClosed {
8237 channels: covered, ..
8238 } => *covered = channels,
8239 }
8240 let body = match serde_json::to_vec(&push) {
8241 Ok(body) => body,
8242 Err(err) => {
8243 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8244 continue;
8245 }
8246 };
8247 let frame = match Frame::build_with_version(
8248 target.negotiated_ver,
8249 FrameType::Push,
8250 control_flags(),
8251 0,
8252 0,
8253 0,
8254 body.clone(),
8255 ) {
8256 Ok(frame) => frame,
8257 Err(err) => {
8258 warn!(
8259 route_channel = target.channel,
8260 error = %err,
8261 "failed to build route lifecycle control PUSH frame"
8262 );
8263 continue;
8264 }
8265 };
8266 if let Err(err) = target.sink.try_send(frame) {
8267 if target.close_on_delivery_failure() {
8268 warn!(
8269 target_connection_id = target.connection_id.get(),
8270 route_channel = target.channel,
8271 error = %err,
8272 "route lifecycle control PUSH was not delivered to client; closing target connection"
8273 );
8274 let _ = forwarding.escalate_client_delivery_failure(
8275 target.connection_id,
8276 target.channel,
8277 target.epoch,
8278 CloseReason::new(
8279 "route_lifecycle_push_delivery_failed",
8280 format!(
8281 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8282 target.channel
8283 ),
8284 ),
8285 crate::forwarding::UndeliveredFrame {
8286 module_id: target.module_id.as_deref(),
8287 sink: &target.sink,
8288 },
8289 );
8290 }
8291 }
8292 }
8293}
8294
8295#[cfg(test)]
8296mod tests {
8297 mod peer_manifest {
8298 include!("peer_manifest_tests.rs");
8299 }
8300 mod activity_read {
8301 include!("activity_read_tests.rs");
8302 }
8303
8304 mod lifecycle_activity {
8305 include!("control_lifecycle_tests.rs");
8306 }
8307 #[cfg(unix)]
8308 #[tokio::test]
8309 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8310 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8311 let path = dir.join("subc.jsonc");
8312 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8313 "program":"/bin/sleep","args":["60"],"protocol":"none",
8314 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8315 }}}).to_string()).unwrap();
8316 let mut configured = crate::daemon_config::load(&path)
8317 .unwrap()
8318 .unwrap()
8319 .modules
8320 .pop()
8321 .unwrap();
8322 let registry = std::sync::Arc::new(crate::Registry::default());
8323 let handle = crate::SupervisorHandle::new();
8324 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8325 .with_handle(handle.clone());
8326 let module = supervisor
8327 .supervise_configured_with_health(
8328 configured.module_spec(),
8329 true,
8330 configured.health.clone(),
8331 None,
8332 configured.restart,
8333 )
8334 .unwrap();
8335 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8336 let before = module.status().unwrap().pid;
8337 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8338 configured.health.cadence = std::time::Duration::from_secs(3600);
8339 let health_only = handler
8340 .reconcile_supervised_modules(
8341 &supervisor,
8342 vec![configured.clone()],
8343 false,
8344 Principal::Direct,
8345 )
8346 .await
8347 .unwrap();
8348 assert!(
8349 health_only.0.changed_pending_reload.is_empty(),
8350 "health policy is already applied live"
8351 );
8352 assert_eq!(module.status().unwrap().pid, before);
8353 assert_eq!(
8354 module.configuration().unwrap().1.http,
8355 configured.health.http
8356 );
8357 configured.args = vec!["61".into()];
8358 let launch = handler
8359 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8360 .await
8361 .unwrap();
8362 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8363 assert_eq!(
8364 module.status().unwrap().pid,
8365 before,
8366 "a launch edit is stored until reload"
8367 );
8368 module.drain().await.unwrap();
8369 }
8370 use cortexkit_test_support::ScratchDir;
8371 use std::{
8372 collections::BTreeMap,
8373 fmt,
8374 path::PathBuf,
8375 sync::{Arc, Mutex},
8376 time::Duration,
8377 };
8378
8379 use serde_json::{json, Value};
8380 use subc_protocol::{
8381 manifest::{
8382 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
8383 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
8384 },
8385 session::HealthStatus,
8386 FrameType,
8387 };
8388
8389 use super::*;
8390 use crate::{
8391 forwarding::{DataRoute, DataRouteState},
8392 registry::ChannelState,
8393 router::FrameSink,
8394 stderr_tail::DEFAULT_MAX_LINE_BYTES,
8395 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
8396 RouteCtx, Router,
8397 };
8398 use tokio::{
8399 sync::mpsc,
8400 time::{sleep, Instant},
8401 };
8402 use tracing::{
8403 field::{Field, Visit},
8404 Event, Subscriber,
8405 };
8406 use tracing_subscriber::{layer::Context, prelude::*, Layer};
8407
8408 fn fake_aft_stub_path() -> PathBuf {
8427 let mut path = std::env::current_exe().expect("current_exe available in tests");
8428 path.pop(); path.pop(); path.push(if cfg!(windows) {
8431 "fake-aft-stub.exe"
8432 } else {
8433 "fake-aft-stub"
8434 });
8435 assert!(
8436 path.exists(),
8437 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
8438 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
8439 path.display()
8440 );
8441 path
8442 }
8443
8444 fn client_retries(code: &str) -> bool {
8449 subc_protocol::error_codes::is_retryable_route_open(code)
8450 }
8451
8452 #[test]
8463 fn retryability_of_forwarding_codes_matches_the_failure() {
8464 let transient = [
8467 ForwardingError::NoModuleConnection,
8468 ForwardingError::ModuleReloading {
8469 module_id: "m".into(),
8470 },
8471 ForwardingError::StaleModuleEndpoint,
8472 ForwardingError::UnknownReservation {
8473 client_channel: 1,
8474 module_channel: 1,
8475 },
8476 ForwardingError::ConnectionClosing {
8477 connection_id: ConnectionId::new(1),
8478 },
8479 ForwardingError::ClientEgressClosed {
8480 connection_id: ConnectionId::new(1),
8481 },
8482 ForwardingError::ModuleEgressUnavailable {
8483 connection_id: ConnectionId::new(1),
8484 },
8485 ];
8486 for err in transient {
8487 let code = forwarding_error_code(&err);
8488 assert!(
8489 client_retries(code),
8490 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
8491 );
8492 }
8493
8494 let permanent = [
8499 ForwardingError::ConnectionRoleConflict {
8500 connection_id: ConnectionId::new(1),
8501 },
8502 ForwardingError::ClientRouteChannelExhausted {
8503 connection_id: ConnectionId::new(1),
8504 },
8505 ForwardingError::ModuleRouteChannelExhausted {
8506 endpoint: ModuleEndpointId {
8507 connection_id: ConnectionId::new(1),
8508 generation: 1,
8509 },
8510 },
8511 ForwardingError::RelayCorrelationExhausted,
8512 ForwardingError::RouteOpenBuild("x".into()),
8513 ForwardingError::Poisoned,
8514 ];
8515 for err in permanent {
8516 let code = forwarding_error_code(&err);
8517 assert!(
8518 !client_retries(code),
8519 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
8520 );
8521 }
8522 }
8523
8524 #[tokio::test]
8552 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
8553 let handler = ControlHandler::default();
8554 let frame =
8555 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
8556
8557 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
8561 assert_eq!(
8562 stamped,
8563 Principal::Direct,
8564 "a caller that proved nothing must not be stamped as a supervised module"
8565 );
8566
8567 let forged = handler
8572 .route_open_principal(
8573 &frame,
8574 Some(ConsumerIdentity {
8575 module_id: "aft".to_string(),
8576 launch_nonce: "not-a-real-nonce".to_string(),
8577 }),
8578 )
8579 .unwrap();
8580 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
8581 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
8582 }
8583
8584 #[test]
8601 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
8602 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
8603 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
8604 let ClientControlRequest::RouteOpen {
8605 consumer_identity, ..
8606 } = parsed
8607 else {
8608 panic!("route.open body must parse as RouteOpen");
8609 };
8610 assert_eq!(
8611 consumer_identity,
8612 Some(ConsumerIdentity {
8613 module_id: "aft".to_string(),
8614 launch_nonce: "n".to_string(),
8615 }),
8616 "the wire field name must reach the value route_open_principal reads"
8617 );
8618 }
8619
8620 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
8621 ModuleManifest::builder(module_id, "0.1.0")
8622 .protocol_ver(protocol_ver)
8623 .provides(vec![ProviderRole::ToolProvider {
8624 tools: vec![Tool {
8625 name: "read".to_string(),
8626 description: None,
8627 execution_mode: ExecutionMode::Pure,
8628 schema: json!({"type": "object"}),
8629 }],
8630 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
8631 concurrency: Concurrency::ModuleManaged,
8632 emits_push: true,
8633 sub_supervises: true,
8634 }])
8635 .build()
8636 }
8637
8638 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
8639 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
8640 }
8641
8642 fn hello_frame_with_control_ops(
8643 module_id: &str,
8644 protocol_ver: u8,
8645 corr: u64,
8646 control_ops: Option<Vec<String>>,
8647 ) -> Frame {
8648 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
8649 }
8650
8651 fn hello_frame_with_nonce(
8652 module_id: &str,
8653 protocol_ver: u8,
8654 corr: u64,
8655 launch_nonce: Option<&str>,
8656 ) -> Frame {
8657 hello_frame_full(
8658 module_id,
8659 protocol_ver,
8660 corr,
8661 None,
8662 launch_nonce.map(ToOwned::to_owned),
8663 )
8664 }
8665
8666 fn hello_frame_full(
8667 module_id: &str,
8668 protocol_ver: u8,
8669 corr: u64,
8670 control_ops: Option<Vec<String>>,
8671 launch_nonce: Option<String>,
8672 ) -> Frame {
8673 let body = serde_json::to_vec(&ModuleHelloBody {
8674 manifest: manifest(module_id, protocol_ver),
8675 protocol_ver,
8676 control_ops,
8677 launch_nonce,
8678 })
8679 .unwrap();
8680 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8681 }
8682
8683 fn hello_frame_with_rich_ui(
8684 module_id: &str,
8685 protocol_ver: u8,
8686 corr: u64,
8687 launch_nonce: Option<&str>,
8688 rich_ui: Value,
8689 ) -> Frame {
8690 let mut body = serde_json::to_value(ModuleHelloBody {
8691 manifest: manifest(module_id, protocol_ver),
8692 protocol_ver,
8693 control_ops: None,
8694 launch_nonce: launch_nonce.map(str::to_string),
8695 })
8696 .unwrap();
8697 body["manifest"]["rich_ui"] = rich_ui;
8698 Frame::build(
8699 FrameType::Hello,
8700 control_flags(),
8701 0,
8702 0,
8703 corr,
8704 serde_json::to_vec(&body).unwrap(),
8705 )
8706 .unwrap()
8707 }
8708
8709 fn non_routable_hello_frame_with_control_ops(
8710 module_id: &str,
8711 corr: u64,
8712 control_ops: Option<Vec<String>>,
8713 ) -> Frame {
8714 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
8715 manifest.provides.clear();
8716 let body = serde_json::to_vec(&ModuleHelloBody {
8717 manifest,
8718 protocol_ver: PROTOCOL_VERSION,
8719 control_ops,
8720 launch_nonce: None,
8721 })
8722 .unwrap();
8723 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
8724 }
8725
8726 fn capability_grammar_hello_frame(
8727 capabilities: Value,
8728 runtime_computed: Option<Value>,
8729 corr: u64,
8730 ) -> Frame {
8731 let mut body = serde_json::to_value(ModuleHelloBody {
8732 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
8733 protocol_ver: PROTOCOL_VERSION,
8734 control_ops: None,
8735 launch_nonce: None,
8736 })
8737 .expect("HELLO body serializes");
8738 body["manifest"]["capabilities"] = capabilities;
8739 if let Some(runtime_computed) = runtime_computed {
8740 body["runtime_computed"] = runtime_computed;
8741 }
8742 Frame::build(
8743 FrameType::Hello,
8744 control_flags(),
8745 0,
8746 0,
8747 corr,
8748 serde_json::to_vec(&body).expect("HELLO body reserializes"),
8749 )
8750 .expect("HELLO frame builds")
8751 }
8752
8753 fn channel_request(channel: u16, corr: u64) -> Frame {
8754 Frame::build(
8755 FrameType::Request,
8756 Flags::new(true, Priority::Interactive, false),
8757 channel,
8758 0,
8759 corr,
8760 b"opaque".to_vec(),
8761 )
8762 .unwrap()
8763 }
8764
8765 fn route_ctx(
8766 connection_id: ConnectionId,
8767 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
8768 let (tx, rx) = mpsc::channel(8);
8769 (
8770 RouteCtx {
8771 connection_id,
8772 egress: FrameSink::new(tx),
8773 },
8774 rx,
8775 )
8776 }
8777
8778 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
8779 serde_json::from_slice(&frame.body).unwrap()
8780 }
8781
8782 async fn hello_via_sink(
8787 handler: &ControlHandler,
8788 ctx: &RouteCtx,
8789 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
8790 hello: Frame,
8791 ) -> Frame {
8792 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
8793 assert!(
8794 replies.is_empty(),
8795 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
8796 );
8797 let ack = rx
8798 .try_recv()
8799 .expect("HELLO_ACK is queued on the module sink")
8800 .frame;
8801 assert_eq!(ack.header.ty, FrameType::HelloAck);
8802 ack
8803 }
8804
8805 fn parse_error(frame: &Frame) -> Value {
8806 serde_json::from_slice(&frame.body).unwrap()
8807 }
8808
8809 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
8810 serde_json::from_slice(&frame.body).unwrap()
8811 }
8812
8813 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
8814 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
8815 route_channel,
8816 route_epoch: 0,
8817 kind,
8818 })
8819 .unwrap();
8820 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8821 }
8822
8823 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
8824 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
8825 module_id: module_id.to_string(),
8826 })
8827 .unwrap();
8828 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8829 }
8830
8831 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
8832 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
8833 }
8834
8835 fn route_open_frame_with_consumer_capabilities(
8836 corr: u64,
8837 module_id: &str,
8838 project_root: ScratchDir,
8839 consumer_capabilities: Option<Vec<String>>,
8840 ) -> Frame {
8841 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8842 target: RouteTarget::ToolProvider {
8843 module_id: module_id.to_string(),
8844 },
8845 identity: BindIdentity::new(
8846 project_root.path().to_path_buf(),
8847 "unit".to_string(),
8848 "session".to_string(),
8849 ),
8850 consumer_identity: None,
8851 consumer_capabilities,
8852 role_versions: None,
8853 admission_facts: None,
8854 scope: None,
8855 })
8856 .unwrap();
8857 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8858 }
8859
8860 fn route_open_frame_with_role_versions(
8861 corr: u64,
8862 module_id: &str,
8863 project_root: ScratchDir,
8864 role_versions: Option<BTreeMap<String, String>>,
8865 ) -> Frame {
8866 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8867 target: RouteTarget::ToolProvider {
8868 module_id: module_id.to_string(),
8869 },
8870 identity: BindIdentity::new(
8871 project_root.path().to_path_buf(),
8872 "unit".to_string(),
8873 format!("session-{corr}"),
8874 ),
8875 consumer_identity: None,
8876 consumer_capabilities: None,
8877 role_versions,
8878 admission_facts: None,
8879 scope: None,
8880 })
8881 .unwrap();
8882 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8883 }
8884
8885 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
8886 entries
8887 .iter()
8888 .map(|(role, version)| (role.to_string(), version.to_string()))
8889 .collect()
8890 }
8891
8892 fn route_open_frame_with_admission_facts(
8893 corr: u64,
8894 module_id: &str,
8895 project_root: ScratchDir,
8896 consumer_identity: Option<subc_control::ConsumerIdentity>,
8897 facts: Option<Value>,
8898 ) -> Frame {
8899 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
8900 target: RouteTarget::ToolProvider {
8901 module_id: module_id.to_string(),
8902 },
8903 identity: BindIdentity::new(
8904 project_root.path().to_path_buf(),
8905 "unit".to_string(),
8906 format!("session-{corr}"),
8907 ),
8908 consumer_identity,
8909 consumer_capabilities: None,
8910 role_versions: None,
8911 admission_facts: facts,
8912 scope: None,
8913 })
8914 .unwrap();
8915 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
8916 }
8917
8918 #[derive(Clone, Default)]
8919 struct EventCapture {
8920 events: Arc<Mutex<Vec<CapturedEvent>>>,
8921 }
8922
8923 #[derive(Clone, Debug)]
8924 struct CapturedEvent {
8925 target: String,
8926 level: tracing::Level,
8927 fields: BTreeMap<String, String>,
8928 }
8929
8930 impl EventCapture {
8931 fn events(&self) -> Vec<CapturedEvent> {
8932 self.events.lock().unwrap().clone()
8933 }
8934 }
8935
8936 impl<S> Layer<S> for EventCapture
8937 where
8938 S: Subscriber,
8939 {
8940 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
8941 let mut visitor = EventFieldVisitor::default();
8942 event.record(&mut visitor);
8943 self.events.lock().unwrap().push(CapturedEvent {
8944 target: event.metadata().target().to_string(),
8945 level: *event.metadata().level(),
8946 fields: visitor.fields,
8947 });
8948 }
8949 }
8950
8951 #[derive(Default)]
8952 struct EventFieldVisitor {
8953 fields: BTreeMap<String, String>,
8954 }
8955
8956 impl Visit for EventFieldVisitor {
8957 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
8958 self.fields
8959 .insert(field.name().to_string(), format!("{value:?}"));
8960 }
8961 }
8962
8963 fn health_response(corr: u64, status: HealthStatus) -> Frame {
8964 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
8965 status,
8966 detail: Some("warming".to_string()),
8967 metrics: Some(json!({"queue_depth": 3})),
8968 })
8969 .unwrap();
8970 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8971 }
8972
8973 fn route_bind_ack(corr: u64) -> Frame {
8974 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
8975 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
8976 }
8977
8978 fn unique_project_root(label: &str) -> ScratchDir {
8979 ScratchDir::new(label)
8980 }
8981
8982 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
8983 match parse_route_poll(frame) {
8984 ClientControlResponse::RoutePoll {
8985 status: None,
8986 live: Some(live),
8987 ..
8988 } => assert_eq!(live, expected_live),
8989 other => panic!("unexpected route.poll response: {other:?}"),
8990 }
8991 }
8992
8993 fn bind_liveness_route(
8994 registry: &Registry,
8995 forwarding: &ForwardingTable,
8996 module_id: &str,
8997 ) -> (RouteCtx, u16, u32) {
8998 let module_connection = ConnectionId::new(101);
8999 let client_connection = ConnectionId::new(202);
9000 let registration = registry
9001 .register_with_control_ops(
9002 manifest(module_id, PROTOCOL_VERSION),
9003 PROTOCOL_VERSION,
9004 module_connection,
9005 module_baseline_control_ops(),
9006 )
9007 .unwrap();
9008 let (module_tx, _module_rx) = mpsc::channel(8);
9009 let endpoint = forwarding
9010 .register_module_connection(
9011 module_connection,
9012 module_id.to_string(),
9013 PROTOCOL_VERSION,
9014 manifest_concurrency(®istration.manifest),
9015 FrameSink::new(module_tx),
9016 )
9017 .unwrap();
9018 let (client_ctx, _client_rx) = route_ctx(client_connection);
9019 let pending = forwarding
9020 .begin_route_bind_relay_for_test(
9021 client_connection,
9022 client_ctx.egress.clone(),
9023 1,
9024 module_id,
9025 )
9026 .unwrap();
9027 assert_eq!(pending.endpoint, endpoint);
9028 let route_channel = pending.client_channel;
9029 let route_epoch = pending.client_epoch;
9030 forwarding
9031 .complete_pending_relay(
9032 module_connection,
9033 pending.corr,
9034 RouteBindRelayOutcome::Accepted,
9035 )
9036 .unwrap();
9037 (client_ctx, route_channel, route_epoch)
9038 }
9039
9040 struct FakeProcessLiveness {
9041 live: Option<bool>,
9042 }
9043
9044 impl ModuleProcessLiveness for FakeProcessLiveness {
9045 fn process_live(&self, _module_id: &str) -> Option<bool> {
9046 self.live
9047 }
9048 }
9049
9050 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9051 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9052 {
9053 let registry = Arc::new(Registry::default());
9054 let supervisor_handle = SupervisorHandle::new();
9055 let supervisor = Supervisor::new_for_test(
9056 Arc::clone(®istry),
9057 RestartPolicy::new(1, Duration::from_millis(10)),
9058 )
9059 .with_handle(supervisor_handle.clone());
9060 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9061 let module = supervisor
9062 .spawn(ModuleSpec {
9063 module_id: "stderr-tail-wire".to_string(),
9064 program: fake_aft_stub_path(),
9065 args: Vec::new(),
9066 env: vec![
9067 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9068 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9069 ],
9070 reserved: false,
9071 reserved_prefixes: Vec::new(),
9072 protocol: ModuleProtocol::Subc,
9073 overlap: Default::default(),
9074 })
9075 .unwrap();
9076
9077 let deadline = Instant::now() + Duration::from_secs(5);
9078 loop {
9079 let tail = module.stderr_tail(None, None);
9080 if tail
9081 .entries
9082 .iter()
9083 .any(|entry| matches!(entry, TailEntry::ProcessStart))
9084 && tail.entries.iter().any(|entry| {
9085 matches!(
9086 entry,
9087 TailEntry::Line {
9088 truncated: true,
9089 ..
9090 }
9091 )
9092 })
9093 {
9094 break;
9095 }
9096 assert!(
9097 Instant::now() < deadline,
9098 "module did not produce a truncated line and restart boundary: {tail:?}"
9099 );
9100 sleep(Duration::from_millis(10)).await;
9101 }
9102
9103 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9104 let request = ClientControlRequest::SupervisorStderrTail {
9105 module_id: "stderr-tail-wire".to_string(),
9106 max_lines: None,
9107 max_bytes: None,
9108 };
9109 let frame = Frame::build(
9110 FrameType::Request,
9111 control_flags(),
9112 0,
9113 0,
9114 1,
9115 serde_json::to_vec(&request).unwrap(),
9116 )
9117 .unwrap();
9118 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9119 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9120 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9121 serde_json::from_slice(&responses[0].body).unwrap()
9122 else {
9123 panic!("expected supervisor.stderr_tail response");
9124 };
9125
9126 assert!(
9127 tail.entries
9128 .iter()
9129 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9130 "the control response lost the restart boundary"
9131 );
9132 let Some(StderrTailEntry::Line {
9133 text,
9134 truncated,
9135 at_ms,
9136 }) = tail.entries.iter().find(|entry| {
9137 matches!(
9138 entry,
9139 StderrTailEntry::Line {
9140 truncated: true,
9141 ..
9142 }
9143 )
9144 })
9145 else {
9146 panic!("the control response lost the truncated line");
9147 };
9148 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9149 assert!(*truncated);
9150 assert!(
9151 at_ms.is_some(),
9152 "the control response lost the line's capture time"
9153 );
9154 }
9155
9156 #[tokio::test(flavor = "current_thread")]
9161 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9162 let dir = ScratchDir::new("terminals-off-worker");
9163 let journal_path = dir.join("terminals.jsonl");
9164 let registry = Arc::new(Registry::default());
9165 let supervisor_handle = SupervisorHandle::new();
9166 let supervisor =
9167 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9168 .with_handle(supervisor_handle.clone())
9169 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9170 let module = supervisor
9171 .spawn(ModuleSpec {
9172 module_id: "terminal-off-worker".to_string(),
9173 program: fake_aft_stub_path(),
9174 args: Vec::new(),
9175 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9176 reserved: false,
9177 reserved_prefixes: Vec::new(),
9178 protocol: ModuleProtocol::Subc,
9179 overlap: Default::default(),
9180 })
9181 .unwrap();
9182 let deadline = Instant::now() + Duration::from_secs(5);
9183 while module.terminal_history().entries.len() != 2 {
9184 assert!(Instant::now() < deadline, "module did not record two exits");
9185 sleep(Duration::from_millis(10)).await;
9186 }
9187
9188 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9189 let handler =
9190 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
9191 let frame = Frame::build(
9192 FrameType::Request,
9193 control_flags(),
9194 0,
9195 0,
9196 1,
9197 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9198 module_id: "terminal-off-worker".to_string(),
9199 })
9200 .unwrap(),
9201 )
9202 .unwrap();
9203 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9204 let spawned_at = std::time::Instant::now();
9205 let read = tokio::spawn({
9206 let handler = Arc::clone(&handler);
9207 async move { handler.handle_control_frame(&ctx, frame).await }
9208 });
9209 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9212 .await
9213 .unwrap()
9214 .expect("the history read reached its pause");
9215 let elapsed = spawned_at.elapsed();
9216 assert!(
9217 elapsed < Duration::from_secs(2) && !read.is_finished(),
9218 "this task could not run while the history read was paused \
9219 (resumed after {elapsed:?}, read finished: {})",
9220 read.is_finished()
9221 );
9222
9223 drop(release);
9224 let responses = read.await.unwrap().unwrap();
9225 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9226 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9227 panic!("expected supervisor.terminals response");
9228 };
9229 assert_eq!(terminals.entries.len(), 2);
9230 assert_eq!(terminals.journal_skipped_lines, 0);
9231 assert_eq!(terminals.journal_read_errors, 0);
9232 }
9233
9234 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9235 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9236 let registry = Arc::new(Registry::default());
9237 let supervisor_handle = SupervisorHandle::new();
9238 let supervisor =
9239 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9240 .with_handle(supervisor_handle.clone());
9241 let module = supervisor
9242 .spawn(ModuleSpec {
9243 module_id: "terminal-golden".to_string(),
9244 program: fake_aft_stub_path(),
9245 args: Vec::new(),
9246 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9247 reserved: false,
9248 reserved_prefixes: Vec::new(),
9249 protocol: ModuleProtocol::Subc,
9250 overlap: Default::default(),
9251 })
9252 .unwrap();
9253
9254 let deadline = Instant::now() + Duration::from_secs(5);
9255 while module.terminal_history().entries.len() != 2 {
9256 assert!(
9257 Instant::now() < deadline,
9258 "module did not retain two terminal exits: {:?}",
9259 module.terminal_history()
9260 );
9261 sleep(Duration::from_millis(10)).await;
9262 }
9263
9264 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9265 let request = ClientControlRequest::SupervisorTerminals {
9266 module_id: "terminal-golden".to_string(),
9267 };
9268 let frame = Frame::build(
9269 FrameType::Request,
9270 control_flags(),
9271 0,
9272 0,
9273 1,
9274 serde_json::to_vec(&request).unwrap(),
9275 )
9276 .unwrap();
9277 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9278 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9279 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9280 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9281 panic!("expected supervisor.terminals response");
9282 };
9283 assert_eq!(terminals.entries.len(), 2);
9284 assert_eq!(terminals.dropped, 0);
9285
9286 let mut rendered = serde_json::to_value(response).unwrap();
9287 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9290 for (index, entry) in rendered["entries"]
9291 .as_array_mut()
9292 .expect("terminal response entries array")
9293 .iter_mut()
9294 .enumerate()
9295 {
9296 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9297 }
9298
9299 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9300 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9301 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9302 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9303 std::fs::write(&golden_path, &serialized).unwrap();
9304 }
9305 let expected: Value =
9306 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9307 assert_eq!(rendered, expected);
9308 }
9309
9310 #[test]
9311 fn hello_registers_manifest_and_returns_ack() {
9312 let registry = Arc::new(Registry::default());
9313 let handler = ControlHandler::new(Arc::clone(®istry));
9314 let conn = ConnectionId::new(1);
9315
9316 let responses = handler
9317 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9318 .unwrap();
9319
9320 assert_eq!(responses.len(), 1);
9321 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9322 assert_eq!(responses[0].header.channel, 0);
9323 assert_eq!(responses[0].header.corr, 7);
9324 let ack = parse_ack(&responses[0]);
9325 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9326 assert!(ack
9327 .subc_capabilities
9328 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9329 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9330 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9331 assert!(ack
9332 .subc_ops
9333 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9334 assert!(ack
9335 .subc_ops
9336 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9337
9338 let registration = registry.get_module("aft").unwrap().unwrap();
9339 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9340 assert_eq!(registration.state, ChannelState::Active);
9341 assert_eq!(registration.connection_id, conn);
9342 assert_eq!(registration.control_ops, module_baseline_control_ops());
9343 }
9344
9345 #[test]
9346 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
9347 let invalid_identifiers = [
9348 ("case_change", "credentials-Provider/v1"),
9349 ("leading_zero", "credentials-provider/v01"),
9350 ("trailing_hyphen", "credentials-provider-/v1"),
9351 ("consecutive_hyphens", "credentials--provider/v1"),
9352 ("uppercase", "Credentials-provider/v1"),
9353 ("missing_v", "credentials-provider/1"),
9354 ("whitespace", "credentials provider/v1"),
9355 ("zero_version", "credentials-provider/v0"),
9356 ("out_of_range_version", "credentials-provider/v4294967296"),
9357 (
9358 "overlength_name",
9359 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
9360 ),
9361 ];
9362 let mut cases = invalid_identifiers
9363 .into_iter()
9364 .map(|(name, identifier)| {
9365 (
9366 format!("identifier_{name}"),
9367 "capabilities.provides[0]".to_string(),
9368 identifier.to_string(),
9369 json!({ "provides": [identifier] }),
9370 None,
9371 )
9372 })
9373 .collect::<Vec<_>>();
9374 cases.extend([
9375 (
9376 "unknown_need".to_string(),
9377 "capabilities.requires[0].need".to_string(),
9378 "deferred".to_string(),
9379 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
9380 None,
9381 ),
9382 (
9383 "duplicate_provides".to_string(),
9384 "capabilities.provides[1]".to_string(),
9385 "credentials-provider/v1".to_string(),
9386 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
9387 None,
9388 ),
9389 (
9390 "duplicate_must_never_reach".to_string(),
9391 "capabilities.must_never_reach[1]".to_string(),
9392 "credentials-provider/v1".to_string(),
9393 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
9394 None,
9395 ),
9396 (
9397 "duplicate_requires_same_need".to_string(),
9398 "capabilities.requires[1]".to_string(),
9399 "credentials-provider/v1".to_string(),
9400 json!({ "requires": [
9401 { "capability": "credentials-provider/v1", "need": "required" },
9402 { "capability": "credentials-provider/v1", "need": "required" }
9403 ] }),
9404 None,
9405 ),
9406 (
9407 "duplicate_requires_conflicting_need".to_string(),
9408 "capabilities.requires[1]".to_string(),
9409 "credentials-provider/v1".to_string(),
9410 json!({ "requires": [
9411 { "capability": "credentials-provider/v1", "need": "required" },
9412 { "capability": "credentials-provider/v1", "need": "optional" }
9413 ] }),
9414 None,
9415 ),
9416 (
9417 "capabilities_root_pointer".to_string(),
9418 "runtime_computed[0]".to_string(),
9419 "/capabilities".to_string(),
9420 json!({}),
9421 Some(json!(["/capabilities"])),
9422 ),
9423 (
9424 "capabilities_descendant_pointer".to_string(),
9425 "runtime_computed[0]".to_string(),
9426 "/capabilities/provides".to_string(),
9427 json!({}),
9428 Some(json!(["/capabilities/provides"])),
9429 ),
9430 (
9431 "malformed_pointer_without_leading_slash".to_string(),
9432 "runtime_computed[0]".to_string(),
9433 "capabilities".to_string(),
9434 json!({}),
9435 Some(json!(["capabilities"])),
9436 ),
9437 (
9438 "malformed_pointer_escape".to_string(),
9439 "runtime_computed[0]".to_string(),
9440 "/roles/~2/tools".to_string(),
9441 json!({}),
9442 Some(json!(["/roles/~2/tools"])),
9443 ),
9444 (
9445 "unknown_capabilities_field".to_string(),
9446 "capabilities.future".to_string(),
9447 "<array>".to_string(),
9448 json!({ "future": [] }),
9449 None,
9450 ),
9451 ]);
9452
9453 for (index, (name, field, value, capabilities, runtime_computed)) in
9454 cases.into_iter().enumerate()
9455 {
9456 let registry = Arc::new(Registry::default());
9457 let handler = ControlHandler::new(Arc::clone(®istry));
9458 let response = handler
9459 .handle_control(
9460 ConnectionId::new((index + 1) as u64),
9461 capability_grammar_hello_frame(
9462 capabilities,
9463 runtime_computed,
9464 index as u64 + 1,
9465 ),
9466 )
9467 .expect("invalid HELLO returns a refusal");
9468
9469 assert_eq!(response.len(), 1, "{name} must emit one refusal");
9470 let error = parse_error(&response[0]);
9471 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
9472 let message = error["message"]
9473 .as_str()
9474 .expect("error message is a string");
9475 assert!(
9476 message.contains(&field),
9477 "{name}: field missing from {message}"
9478 );
9479 assert!(
9480 message.contains(&value),
9481 "{name}: value missing from {message}"
9482 );
9483 assert_eq!(
9484 registry
9485 .active_registration_count()
9486 .expect("registry reads"),
9487 0,
9488 "{name}: refused HELLO must not create a catalog entry"
9489 );
9490 }
9491 }
9492
9493 #[test]
9494 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
9495 let registry = Arc::new(Registry::default());
9496 let handler = ControlHandler::new(Arc::clone(®istry));
9497 let capabilities = json!({
9498 "provides": ["credentials-provider/v1"],
9499 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
9500 "must_never_reach": ["federation-transport/v1"]
9501 });
9502 let response = handler
9503 .handle_control(
9504 ConnectionId::new(99),
9505 capability_grammar_hello_frame(
9506 capabilities.clone(),
9507 Some(json!(["/roles/0/tools"])),
9508 99,
9509 ),
9510 )
9511 .expect("valid HELLO registers");
9512 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9513
9514 let request = Frame::build(
9515 FrameType::Request,
9516 control_flags(),
9517 0,
9518 0,
9519 100,
9520 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9521 .expect("catalog request serializes"),
9522 )
9523 .expect("catalog request frame builds");
9524 let response = handler
9525 .handle_catalog_list(request, None)
9526 .expect("catalog list succeeds");
9527 let ClientControlResponse::CatalogList { modules, .. } =
9528 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
9529 else {
9530 panic!("catalog request must return catalog.list");
9531 };
9532 assert_eq!(modules.len(), 1);
9533 assert_eq!(
9534 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
9535 capabilities
9536 );
9537 }
9538
9539 #[test]
9540 fn catalog_list_mirrors_management_operation_description() {
9541 let registry = Arc::new(Registry::default());
9542 let handler = ControlHandler::new(Arc::clone(®istry));
9543 let description = "List managed records and return their identifiers and metadata.";
9544 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
9545 manifest.provides = vec![ProviderRole::ManagementSurface {
9546 operations: vec![ManagementOperation {
9547 name: "records.list".to_string(),
9548 kind: ManagementOperationKind::Query,
9549 description: Some(description.to_string()),
9550 }],
9551 config_schema: json!({"type": "object"}),
9552 observability: vec![ObservabilitySurface {
9553 name: "records.stats".to_string(),
9554 kind: ObservabilityKind::Snapshot,
9555 }],
9556 identity_scope: vec![IdentityScope::Project],
9557 concurrency: Concurrency::ModuleManaged,
9558 }];
9559 registry
9560 .register_with_control_ops(
9561 manifest,
9562 PROTOCOL_VERSION,
9563 ConnectionId::new(99),
9564 Vec::new(),
9565 )
9566 .expect("described management manifest registers");
9567
9568 let request = Frame::build(
9569 FrameType::Request,
9570 control_flags(),
9571 0,
9572 0,
9573 100,
9574 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
9575 .expect("catalog request serializes"),
9576 )
9577 .expect("catalog request frame builds");
9578 let response = handler
9579 .handle_catalog_list(request, None)
9580 .expect("catalog list succeeds");
9581 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
9582 assert_eq!(
9583 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
9584 "catalog.list must preserve the declared operation description verbatim"
9585 );
9586 }
9587
9588 #[test]
9589 fn catalog_list_mirrors_registered_operation_delivery() {
9590 let registry = Arc::new(Registry::default());
9591 let handler = ControlHandler::new(Arc::clone(®istry));
9592 let delivery =
9593 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
9594 let mut body = serde_json::to_value(ModuleHelloBody {
9595 manifest: manifest("watch-management", PROTOCOL_VERSION),
9596 protocol_ver: PROTOCOL_VERSION,
9597 control_ops: None,
9598 launch_nonce: None,
9599 })
9600 .unwrap();
9601 body["manifest"]["provides"] = json!([{
9602 "role": "management_surface",
9603 "operations": [
9604 {"name": "status", "kind": "query"},
9605 {"name": "future", "kind": "query"},
9606 {"name": "reset", "kind": "mutate"}
9607 ],
9608 "config_schema": {}, "observability": [], "identity_scope": []
9609 }]);
9610 body["manifest"]["operation_delivery"] = delivery.clone();
9611 let hello = Frame::build(
9612 FrameType::Hello,
9613 control_flags(),
9614 0,
9615 0,
9616 99,
9617 serde_json::to_vec(&body).unwrap(),
9618 )
9619 .unwrap();
9620 let response = handler
9621 .handle_control(ConnectionId::new(99), hello)
9622 .unwrap();
9623 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9624
9625 let (_, registrations) = registry.list_modules().unwrap();
9626 let registered = ®istrations[0].manifest;
9627 assert!(registered.is_watch_operation("status"));
9628 assert!(!registered.is_watch_operation("future"));
9629 assert!(!registered.is_watch_operation("reset"));
9630 assert!(!registered.is_watch_operation("missing"));
9631
9632 let request = Frame::build(
9633 FrameType::Request,
9634 control_flags(),
9635 0,
9636 0,
9637 100,
9638 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9639 )
9640 .unwrap();
9641 let response = handler.handle_catalog_list(request, None).unwrap();
9642 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
9643 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
9644 let ClientControlResponse::CatalogList { modules, .. } =
9645 serde_json::from_slice(&response[0].body).unwrap()
9646 else {
9647 panic!("expected catalog.list");
9648 };
9649 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
9650 }
9651
9652 #[test]
9653 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
9654 let registry = Arc::new(Registry::default());
9655 let supervisor = SupervisorHandle::new();
9656 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
9657 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
9658 let descriptor = json!({
9659 "schema": "ckios.richui.manifest/1",
9660 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
9661 "bytes": 4096,
9662 "chunks": 4
9663 });
9664 let connection = ConnectionId::new(201);
9665 let hello = handler
9666 .handle_control(
9667 connection,
9668 hello_frame_with_rich_ui(
9669 "vault",
9670 PROTOCOL_VERSION,
9671 201,
9672 Some("the-real-nonce"),
9673 descriptor.clone(),
9674 ),
9675 )
9676 .unwrap();
9677 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
9678
9679 let request =
9680 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
9681 let provides = registry
9682 .get_module_by_connection(connection)
9683 .unwrap()
9684 .unwrap()
9685 .manifest
9686 .provides;
9687 handler
9688 .handle_catalog_update(connection, request, provides, None, Some(false))
9689 .unwrap();
9690
9691 let request = Frame::build(
9692 FrameType::Request,
9693 control_flags(),
9694 0,
9695 0,
9696 203,
9697 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9698 )
9699 .unwrap();
9700 let response = handler.handle_catalog_list(request, None).unwrap();
9701 let ClientControlResponse::CatalogList { modules, .. } =
9702 serde_json::from_slice(&response[0].body).unwrap()
9703 else {
9704 panic!("expected catalog.list");
9705 };
9706 assert_eq!(
9707 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
9708 descriptor
9709 );
9710 assert!(modules[0].nonce_verified);
9711 }
9712
9713 fn hello_with_role_declarations(declarations: Value) -> Frame {
9714 let mut body = serde_json::to_value(ModuleHelloBody {
9715 manifest: manifest("role-test", PROTOCOL_VERSION),
9716 protocol_ver: PROTOCOL_VERSION,
9717 control_ops: None,
9718 launch_nonce: None,
9719 })
9720 .unwrap();
9721 body["manifest"]["role_declarations"] = declarations;
9722 Frame::build(
9723 FrameType::Hello,
9724 control_flags(),
9725 0,
9726 0,
9727 301,
9728 serde_json::to_vec(&body).unwrap(),
9729 )
9730 .unwrap()
9731 }
9732
9733 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
9734 let request = Frame::build(
9735 FrameType::Request,
9736 control_flags(),
9737 0,
9738 0,
9739 302,
9740 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
9741 )
9742 .unwrap();
9743 let response = handler.handle_catalog_list(request, None).unwrap();
9744 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
9745 let ClientControlResponse::CatalogList { mut modules, .. } =
9746 serde_json::from_slice(&response[0].body).unwrap()
9747 else {
9748 panic!("expected catalog.list");
9749 };
9750 (raw["modules"][0].clone(), modules.remove(0))
9751 }
9752
9753 #[test]
9754 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
9755 let registry = Arc::new(Registry::default());
9756 let handler = ControlHandler::new(Arc::clone(®istry));
9757 let declarations = json!({
9758 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
9759 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
9760 });
9761 let connection = ConnectionId::new(301);
9762 let response = handler
9763 .handle_control(
9764 connection,
9765 hello_with_role_declarations(declarations.clone()),
9766 )
9767 .unwrap();
9768 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9769 let registered = registry
9770 .get_module_by_connection(connection)
9771 .unwrap()
9772 .unwrap()
9773 .manifest;
9774 assert_eq!(
9778 serde_json::to_value(®istered.role_declarations).unwrap(),
9779 declarations
9780 );
9781 let (raw, entry) = role_declarations_catalog(&handler);
9782 assert_eq!(raw["role_declarations"], declarations);
9783 assert_eq!(entry.role_declarations, registered.role_declarations);
9784
9785 let request =
9786 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
9787 handler
9788 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
9789 .unwrap();
9790 let (raw, entry) = role_declarations_catalog(&handler);
9791 assert_eq!(raw["role_declarations"], declarations);
9792 assert_eq!(entry.role_declarations, registered.role_declarations);
9793 }
9794
9795 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
9796 let registry = Arc::new(Registry::default());
9797 let handler = ControlHandler::new(Arc::clone(®istry));
9798 let capture = EventCapture::default();
9799 let _subscriber =
9800 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9801 let response = handler
9802 .handle_control(
9803 ConnectionId::new(301),
9804 hello_with_role_declarations(declarations),
9805 )
9806 .unwrap();
9807 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9808 assert!(registry
9809 .get_module_by_connection(ConnectionId::new(301))
9810 .unwrap()
9811 .unwrap()
9812 .manifest
9813 .role_declarations
9814 .is_empty());
9815 let (raw, entry) = role_declarations_catalog(&handler);
9816 assert!(raw.get("role_declarations").is_none());
9817 assert!(entry.role_declarations.is_empty());
9818 let warnings: Vec<_> = capture
9819 .events()
9820 .into_iter()
9821 .filter(|event| {
9822 event.fields.get("message").map(String::as_str)
9823 == Some("dropping whole role_declarations map from HELLO")
9824 })
9825 .collect();
9826 assert_eq!(warnings.len(), 1);
9827 assert_eq!(warnings[0].level, tracing::Level::WARN);
9828 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
9829 assert_eq!(warnings[0].fields["reason"], reason);
9830 }
9831
9832 #[test]
9833 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
9834 for value in [
9835 json!(null),
9836 json!(false),
9837 json!(0),
9838 json!("opaque"),
9839 json!([]),
9840 ] {
9841 assert_hello_drops_role_declarations_with_warning(
9842 json!({"routing/v1": {}, "broken/v1": value}),
9843 "role_declarations values must be objects",
9844 );
9845 }
9846 assert_hello_drops_role_declarations_with_warning(
9847 json!([]),
9848 "role_declarations must be an object",
9849 );
9850 }
9851
9852 #[test]
9853 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
9854 assert_hello_drops_role_declarations_with_warning(
9855 json!({"routing/v1": {}, "bad key": {}}),
9856 "role_declarations key must use capability-name grammar",
9857 );
9858 }
9859
9860 #[test]
9861 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
9862 assert_hello_drops_role_declarations_with_warning(
9863 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
9864 "role_declarations exceeds 16384 compact JSON bytes",
9865 );
9866 }
9867
9868 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
9869 let mut body = serde_json::to_value(ModuleHelloBody {
9870 manifest: manifest("flow-ops", PROTOCOL_VERSION),
9871 protocol_ver: PROTOCOL_VERSION,
9872 control_ops: None,
9873 launch_nonce: None,
9874 })
9875 .unwrap();
9876 body["manifest"]["provides"] = json!([{
9877 "role": "management_surface",
9878 "operations": [
9879 {"name": "diagnostics.findings", "kind": "query"},
9880 {"name": "reset", "kind": "mutate"}
9881 ],
9882 "config_schema": {}, "observability": [], "identity_scope": []
9883 }]);
9884 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
9885 if let Some(flow_operations) = flow_operations {
9886 body["manifest"]["flow_operations"] = flow_operations;
9887 }
9888 Frame::build(
9889 FrameType::Hello,
9890 control_flags(),
9891 0,
9892 0,
9893 401,
9894 serde_json::to_vec(&body).unwrap(),
9895 )
9896 .unwrap()
9897 }
9898
9899 #[test]
9900 fn catalog_list_mirrors_flow_operations_unchanged() {
9901 for (capability, flow_operations) in [
9904 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
9905 ("call-modes/v1", json!(["diagnostics.findings"])),
9906 ("flow-scopes/v1", json!([])),
9907 ] {
9908 let registry = Arc::new(Registry::default());
9909 let handler = ControlHandler::new(Arc::clone(®istry));
9910 let connection = ConnectionId::new(401);
9911 let response = handler
9912 .handle_control(
9913 connection,
9914 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
9915 )
9916 .unwrap();
9917 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9918 let registered = registry
9919 .get_module_by_connection(connection)
9920 .unwrap()
9921 .unwrap()
9922 .manifest;
9923 assert_eq!(
9924 serde_json::to_value(®istered.flow_operations).unwrap(),
9925 flow_operations
9926 );
9927 let (raw, entry) = role_declarations_catalog(&handler);
9928 assert_eq!(raw["flow_operations"], flow_operations);
9929 assert_eq!(entry.flow_operations, registered.flow_operations);
9930 }
9931
9932 let registry = Arc::new(Registry::default());
9933 let handler = ControlHandler::new(Arc::clone(®istry));
9934 let response = handler
9935 .handle_control(
9936 ConnectionId::new(401),
9937 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
9938 )
9939 .unwrap();
9940 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9941 let (raw, entry) = role_declarations_catalog(&handler);
9942 assert!(raw.get("flow_operations").is_none());
9943 assert_eq!(entry.flow_operations, None);
9944 }
9945
9946 fn assert_hello_drops_flow_operations_with_warning(
9947 capabilities: Value,
9948 flow_operations: Value,
9949 reason: &str,
9950 ) {
9951 let registry = Arc::new(Registry::default());
9952 let handler = ControlHandler::new(Arc::clone(®istry));
9953 let capture = EventCapture::default();
9954 let _subscriber =
9955 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
9956 let response = handler
9957 .handle_control(
9958 ConnectionId::new(401),
9959 hello_with_flow_operations(capabilities, Some(flow_operations)),
9960 )
9961 .unwrap();
9962 assert_eq!(response[0].header.ty, FrameType::HelloAck);
9963 let registered = registry
9964 .get_module_by_connection(ConnectionId::new(401))
9965 .unwrap()
9966 .expect("the module still registers")
9967 .manifest;
9968 assert_eq!(registered.module_id, "flow-ops");
9969 assert_eq!(registered.flow_operations, None);
9970 let (raw, entry) = role_declarations_catalog(&handler);
9971 assert_eq!(raw["module_id"], "flow-ops");
9972 assert!(raw.get("flow_operations").is_none());
9973 assert_eq!(entry.flow_operations, None);
9974 let warnings: Vec<_> = capture
9975 .events()
9976 .into_iter()
9977 .filter(|event| {
9978 event.fields.get("message").map(String::as_str)
9979 == Some("dropping flow_operations from HELLO")
9980 })
9981 .collect();
9982 assert_eq!(warnings.len(), 1);
9983 assert_eq!(warnings[0].level, tracing::Level::WARN);
9984 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
9985 assert_eq!(warnings[0].fields["reason"], reason);
9986 }
9987
9988 #[test]
9989 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
9990 assert_hello_drops_flow_operations_with_warning(
9991 json!(["flow-scopes/v1"]),
9992 json!(["diagnostics.findings", "missing"]),
9993 "flow_operations names 'missing', which is not a declared management operation",
9994 );
9995 }
9996
9997 #[test]
9998 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
9999 assert_hello_drops_flow_operations_with_warning(
10000 json!([]),
10001 json!(["diagnostics.findings"]),
10002 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10003 );
10004 assert_hello_drops_flow_operations_with_warning(
10006 json!(["agent-run-scopes/v1"]),
10007 json!([]),
10008 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10009 );
10010 }
10011
10012 #[test]
10013 fn hello_malformed_flow_operations_is_dropped_with_warning() {
10014 assert_hello_drops_flow_operations_with_warning(
10015 json!(["flow-scopes/v1"]),
10016 json!(["reset", "reset"]),
10017 "flow_operations names 'reset' more than once",
10018 );
10019 assert_hello_drops_flow_operations_with_warning(
10020 json!(["flow-scopes/v1"]),
10021 json!(vec!["reset"; 257]),
10022 "flow_operations lists 257 names, more than 256",
10023 );
10024 assert_hello_drops_flow_operations_with_warning(
10025 json!(["flow-scopes/v1"]),
10026 json!("reset"),
10027 "flow_operations must be an array of operation names",
10028 );
10029 }
10030
10031 #[test]
10032 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
10033 let registry = Arc::new(Registry::default());
10034 let handler = ControlHandler::new(registry.clone());
10035 let descriptor = json!({
10036 "schema": "ckios.richui.manifest/1",
10037 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10038 "bytes": 128,
10039 "chunks": 2
10040 });
10041 let malformed = [
10042 ("schema", json!("")),
10043 ("schema", json!("a".repeat(65))),
10044 ("schema", json!("Bad schema")),
10045 (
10046 "manifest_digest",
10047 json!(format!("sha256:{}", "A".repeat(64))),
10048 ),
10049 ("manifest_digest", json!("sha256:abc")),
10050 ("bytes", json!(0)),
10051 ("bytes", json!(16_777_217)),
10052 ("bytes", json!("128")),
10053 ("chunks", json!(0)),
10054 ("chunks", json!(1025)),
10055 ("chunks", json!("2")),
10056 ];
10057 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
10058 let mut invalid = descriptor.clone();
10059 invalid[field] = invalid_value;
10060 let response = handler
10061 .handle_control(
10062 ConnectionId::new(210 + index as u64),
10063 hello_frame_with_rich_ui(
10064 &format!("bad-rich-ui-{index}"),
10065 PROTOCOL_VERSION,
10066 210 + index as u64,
10067 None,
10068 invalid,
10069 ),
10070 )
10071 .unwrap();
10072 assert_eq!(response[0].header.ty, FrameType::Error);
10073 let error = parse_error(&response[0]);
10074 assert_eq!(error["code"], "invalid_manifest");
10075 assert!(error["message"]
10076 .as_str()
10077 .unwrap()
10078 .contains(&format!("rich_ui.{field}")));
10079 assert!(registry
10080 .get_module(&format!("bad-rich-ui-{index}"))
10081 .unwrap()
10082 .is_none());
10083 }
10084 }
10085
10086 #[test]
10087 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
10088 let registry = Arc::new(Registry::default());
10089 let handler = ControlHandler::new(Arc::clone(®istry));
10090 let connection = ConnectionId::new(219);
10091 registry
10092 .register_with_control_ops(
10093 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
10094 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
10095 "bad schema",
10096 format!("sha256:{}", "a".repeat(64)),
10097 128,
10098 2,
10099 )))
10100 .build(),
10101 PROTOCOL_VERSION,
10102 connection,
10103 Vec::new(),
10104 )
10105 .unwrap();
10106 let request =
10107 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
10108 let response = handler
10109 .handle_catalog_update(connection, request, Vec::new(), None, None)
10110 .unwrap();
10111 assert_eq!(response[0].header.ty, FrameType::Error);
10112 let error = parse_error(&response[0]);
10113 assert_eq!(error["code"], "invalid_manifest");
10114 assert!(error["message"]
10115 .as_str()
10116 .unwrap()
10117 .contains("rich_ui.schema"));
10118 }
10119
10120 #[test]
10121 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
10122 let registry = Arc::new(Registry::default());
10123 let supervisor = SupervisorHandle::new();
10124 supervisor.open_swap("provider", "candidate-nonce".to_string());
10125 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10126 let connection = ConnectionId::new(218);
10127 let hello = handler
10128 .handle_control(
10129 connection,
10130 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
10131 )
10132 .unwrap();
10133 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10134 assert!(registry.list_modules().unwrap().1.is_empty());
10135
10136 registry.promote_candidate("provider").unwrap().unwrap();
10137 let request = Frame::build(
10138 FrameType::Request,
10139 control_flags(),
10140 0,
10141 0,
10142 219,
10143 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10144 )
10145 .unwrap();
10146 let response = handler.handle_catalog_list(request, None).unwrap();
10147 let ClientControlResponse::CatalogList { modules, .. } =
10148 serde_json::from_slice(&response[0].body).unwrap()
10149 else {
10150 panic!("expected catalog.list");
10151 };
10152 assert_eq!(modules.len(), 1);
10153 assert!(modules[0].nonce_verified);
10154 }
10155
10156 #[test]
10157 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
10158 let registry = Arc::new(Registry::default());
10159 let handler = ControlHandler::new(Arc::clone(®istry));
10160 let hello = Frame::build(
10161 FrameType::Hello,
10162 control_flags(),
10163 0,
10164 0,
10165 220,
10166 serde_json::to_vec(&ModuleHelloBody {
10167 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
10168 protocol_ver: PROTOCOL_VERSION,
10169 control_ops: None,
10170 launch_nonce: Some("unverified".to_string()),
10171 })
10172 .unwrap(),
10173 )
10174 .unwrap();
10175 assert_eq!(
10176 handler
10177 .handle_control(ConnectionId::new(220), hello)
10178 .unwrap()[0]
10179 .header
10180 .ty,
10181 FrameType::HelloAck
10182 );
10183 let request = Frame::build(
10184 FrameType::Request,
10185 control_flags(),
10186 0,
10187 0,
10188 221,
10189 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10190 )
10191 .unwrap();
10192 let response = handler.handle_catalog_list(request, None).unwrap();
10193 let ClientControlResponse::CatalogList { modules, .. } =
10194 serde_json::from_slice(&response[0].body).unwrap()
10195 else {
10196 panic!("expected catalog.list");
10197 };
10198 assert_eq!(
10199 serde_json::to_vec(&modules[0]).unwrap(),
10200 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
10201 );
10202 assert!(!modules[0].nonce_verified);
10203 }
10204
10205 #[test]
10206 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
10207 let registry = Arc::new(Registry::default());
10208 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10209 [("vault".to_string(), true), ("squatter".to_string(), true)],
10210 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10211 );
10212 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
10213 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10214 provides: vec!["credentials-provider/v1".to_string()],
10215 requires: Vec::new(),
10216 must_never_reach: Vec::new(),
10217 });
10218 let frame = Frame::build(
10219 FrameType::Hello,
10220 control_flags(),
10221 0,
10222 0,
10223 77,
10224 serde_json::to_vec(&ModuleHelloBody {
10225 manifest: squatter,
10226 protocol_ver: PROTOCOL_VERSION,
10227 control_ops: None,
10228 launch_nonce: None,
10229 })
10230 .expect("HELLO serializes"),
10231 )
10232 .expect("HELLO frame builds");
10233 let response = handler
10234 .handle_control(ConnectionId::new(77), frame)
10235 .expect("reserved claim receives a typed refusal");
10236 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10237 assert_eq!(
10238 registry
10239 .active_registration_count()
10240 .expect("registry reads"),
10241 0,
10242 "a reserved capability refusal must not leave a catalog entry"
10243 );
10244 }
10245
10246 #[test]
10247 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10248 for settlement in ["timeout", "inconclusive", "drop"] {
10249 let breakers = RouteBindBreakers::default();
10250 let RouteBindAdmission::Admitted {
10251 guard: mut old,
10252 probe: false,
10253 } = breakers.admit("prov")
10254 else {
10255 panic!("ordinary relay admitted")
10256 };
10257 let RouteBindAdmission::Admitted {
10258 guard: mut opener, ..
10259 } = breakers.admit("prov")
10260 else {
10261 panic!("second relay admitted")
10262 };
10263 assert!(
10264 !opener
10265 .record_timeout(1, Duration::ZERO)
10266 .unwrap()
10267 .reopened_after_probe
10268 );
10269 let RouteBindAdmission::Admitted {
10270 guard: mut probe,
10271 probe: true,
10272 } = breakers.admit("prov")
10273 else {
10274 panic!("one cooldown probe admitted")
10275 };
10276 match settlement {
10277 "timeout" => assert!(
10278 !old.record_timeout(1, Duration::ZERO)
10279 .unwrap()
10280 .reopened_after_probe
10281 ),
10282 "inconclusive" => old.record_inconclusive(),
10283 "drop" => drop(old),
10284 _ => unreachable!(),
10285 }
10286 assert!(
10287 matches!(
10288 breakers.admit("prov"),
10289 RouteBindAdmission::Refused {
10290 probe_in_flight: true,
10291 ..
10292 }
10293 ),
10294 "{settlement} of a pre-open relay cannot release the real probe"
10295 );
10296 assert!(
10297 probe
10298 .record_timeout(1, Duration::ZERO)
10299 .unwrap()
10300 .reopened_after_probe
10301 );
10302 assert!(matches!(
10303 breakers.admit("prov"),
10304 RouteBindAdmission::Admitted { probe: true, .. }
10305 ));
10306 }
10307 let breakers = RouteBindBreakers::default();
10308 let admit = || match breakers.admit("prov") {
10309 RouteBindAdmission::Admitted { guard, .. } => guard,
10310 _ => panic!("relay admitted"),
10311 };
10312 admit().record_timeout(1, Duration::ZERO);
10313 let mut old_probe = admit();
10314 breakers.reset_for_new_module_connection("prov");
10315 admit().record_timeout(1, Duration::ZERO);
10316 let _new_probe = admit();
10317 old_probe.record_inconclusive();
10318 assert!(matches!(
10319 breakers.admit("prov"),
10320 RouteBindAdmission::Refused {
10321 probe_in_flight: true,
10322 ..
10323 }
10324 ));
10325 }
10326
10327 #[tokio::test]
10328 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
10329 for candidate in [false, true] {
10330 let registry = Arc::new(Registry::default());
10331 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10332 [("vault".to_string(), true), ("squatter".to_string(), true)],
10333 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10334 );
10335 let conn = ConnectionId::new(77);
10336 let (ctx, mut rx) = route_ctx(conn);
10337 let initial = capability_manifest("squatter", &[], &[]);
10338 if candidate {
10339 registry
10340 .register_candidate_with_control_ops(
10341 initial.clone(),
10342 PROTOCOL_VERSION,
10343 conn,
10344 module_baseline_control_ops(),
10345 )
10346 .unwrap();
10347 handler
10348 .forwarding
10349 .register_candidate_module_connection(
10350 conn,
10351 "squatter".to_string(),
10352 PROTOCOL_VERSION,
10353 manifest_concurrency(&initial),
10354 ctx.egress.clone(),
10355 )
10356 .unwrap();
10357 } else {
10358 hello_via_sink(
10359 &handler,
10360 &ctx,
10361 &mut rx,
10362 hello_frame_with_manifest(initial.clone(), 1),
10363 )
10364 .await;
10365 }
10366 let response = handler
10367 .handle_control_frame(
10368 &ctx,
10369 catalog_update_with_capabilities_frame(
10370 2,
10371 capability_manifest("squatter", &["credentials-provider/v1"], &[])
10372 .capabilities
10373 .unwrap(),
10374 ),
10375 )
10376 .await
10377 .unwrap();
10378 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10379 assert_eq!(
10380 registry
10381 .get_module_by_connection(conn)
10382 .unwrap()
10383 .unwrap()
10384 .manifest,
10385 initial
10386 );
10387 }
10388 }
10389
10390 #[test]
10391 fn server_describe_surfaces_required_capability_verdict_fields() {
10392 let registry = Arc::new(Registry::default());
10393 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10394 [
10395 ("consumer".to_string(), true),
10396 ("provider".to_string(), false),
10397 ],
10398 BTreeMap::new(),
10399 );
10400 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10401 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10402 provides: Vec::new(),
10403 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10404 capability: "credentials-provider/v1".to_string(),
10405 need: subc_protocol::manifest::CapabilityNeed::Required,
10406 }],
10407 must_never_reach: Vec::new(),
10408 });
10409 let hello = Frame::build(
10410 FrameType::Hello,
10411 control_flags(),
10412 0,
10413 0,
10414 78,
10415 serde_json::to_vec(&ModuleHelloBody {
10416 manifest: consumer,
10417 protocol_ver: PROTOCOL_VERSION,
10418 control_ops: None,
10419 launch_nonce: None,
10420 })
10421 .expect("HELLO serializes"),
10422 )
10423 .expect("HELLO frame builds");
10424 handler
10425 .handle_control(ConnectionId::new(78), hello)
10426 .expect("consumer registers");
10427 let describe = Frame::build(
10428 FrameType::Request,
10429 control_flags(),
10430 0,
10431 0,
10432 79,
10433 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
10434 .expect("request serializes"),
10435 )
10436 .expect("describe frame builds");
10437 let response = handler
10438 .handle_server_describe(describe)
10439 .expect("server.describe succeeds");
10440 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
10441 let requirement = &rendered["capability_requirements"][0];
10442 assert_eq!(requirement["consumer"], "consumer");
10443 assert_eq!(requirement["verdict"], "never_provided");
10444 assert_eq!(requirement["episode_seq"], 1);
10445 assert_eq!(requirement["config_satisfiable"], false);
10446 assert_eq!(requirement["runtime_available"], false);
10447 assert!(requirement["detail"]
10448 .as_str()
10449 .expect("detail string")
10450 .contains("credentials-provider/v1"));
10451 }
10452
10453 fn deregister_required_provider(
10459 daemon_shutdown_begun: bool,
10460 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
10461 const CAPABILITY: &str = "project-identity/v1";
10462 let registry = Arc::new(Registry::default());
10463 let daemon_shutdown = DaemonShutdownFlag::default();
10464 let handler = ControlHandler::new(Arc::clone(®istry))
10465 .with_capability_config(
10466 [
10467 ("consumer".to_string(), true),
10468 ("provider".to_string(), true),
10469 ],
10470 BTreeMap::new(),
10471 )
10472 .with_daemon_shutdown_flag(daemon_shutdown.clone());
10473 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
10474 consumer.capabilities = Some(CapabilityDeclarations {
10475 provides: Vec::new(),
10476 requires: vec![subc_protocol::manifest::CapabilityRequirement {
10477 capability: CAPABILITY.to_string(),
10478 need: subc_protocol::manifest::CapabilityNeed::Required,
10479 }],
10480 must_never_reach: Vec::new(),
10481 });
10482 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
10483 let provider_conn = ConnectionId::new(182);
10484 for (conn, manifest, corr) in [
10485 (ConnectionId::new(181), consumer, 181),
10486 (provider_conn, provider, 182),
10487 ] {
10488 let hello = handler
10489 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
10490 .expect("module registers");
10491 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10492 }
10493 assert_eq!(
10494 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10495 Some(CapabilityVerdict::Provided),
10496 "the registered provider must satisfy the requirement first"
10497 );
10498 if daemon_shutdown_begun {
10499 daemon_shutdown.set_for_test();
10500 }
10501
10502 let capture = EventCapture::default();
10503 let _subscriber =
10504 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10505 handler
10506 .cleanup_connection(provider_conn)
10507 .expect("provider deregisters");
10508 let events = capture
10509 .events()
10510 .into_iter()
10511 .filter(|event| {
10512 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
10513 })
10514 .collect();
10515 (
10516 events,
10517 handler.capability_evaluator.verdict("consumer", CAPABILITY),
10518 )
10519 }
10520
10521 #[test]
10522 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
10523 let (events, verdict) = deregister_required_provider(true);
10524 assert_eq!(
10525 verdict,
10526 Some(CapabilityVerdict::NeverProvided),
10527 "shutdown must not change the verdict route.open refusals read"
10528 );
10529 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10530 let event = &events[0];
10531 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
10532 assert_eq!(
10533 event.fields.get("cause").map(String::as_str),
10534 Some("\"daemon_shutdown\""),
10535 "{event:?}"
10536 );
10537 assert_eq!(
10538 event.fields.get("verdict").map(String::as_str),
10539 Some("\"never_provided\""),
10540 "{event:?}"
10541 );
10542 }
10543
10544 #[test]
10545 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
10546 let (events, verdict) = deregister_required_provider(false);
10547 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
10548 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
10549 let event = &events[0];
10550 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
10551 assert_eq!(event.fields.get("cause"), None, "{event:?}");
10552 assert_eq!(
10553 event.fields.get("verdict").map(String::as_str),
10554 Some("\"never_provided\""),
10555 "{event:?}"
10556 );
10557 }
10558
10559 #[test]
10560 fn catalog_list_omits_capabilities_for_legacy_manifest() {
10561 let registry = Arc::new(Registry::default());
10562 let handler = ControlHandler::new(Arc::clone(®istry));
10563 let hello = handler
10564 .handle_control(
10565 ConnectionId::new(101),
10566 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
10567 )
10568 .expect("legacy HELLO registers");
10569 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10570
10571 let request = Frame::build(
10572 FrameType::Request,
10573 control_flags(),
10574 0,
10575 0,
10576 102,
10577 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10578 .expect("catalog request serializes"),
10579 )
10580 .expect("catalog request frame builds");
10581 let response = handler
10582 .handle_catalog_list(request, None)
10583 .expect("catalog list succeeds");
10584 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10585 assert!(
10586 body["modules"][0].get("capabilities").is_none(),
10587 "legacy manifest must retain an absent capabilities field on catalog.list"
10588 );
10589 }
10590
10591 #[test]
10592 fn hello_ack_omits_storage_when_no_storage_config() {
10593 let registry = Arc::new(Registry::default());
10594 let handler = ControlHandler::new(Arc::clone(®istry));
10595 let responses = handler
10596 .handle_control(
10597 ConnectionId::new(1),
10598 hello_frame("aft", PROTOCOL_VERSION, 7),
10599 )
10600 .unwrap();
10601 let ack = parse_ack(&responses[0]);
10602 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
10603 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
10604 }
10605
10606 #[tokio::test]
10607 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
10608 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
10609 let registry = Arc::new(Registry::default());
10610 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
10611 let responses = handler
10612 .handle_control(
10613 ConnectionId::new(1),
10614 hello_frame("aft", PROTOCOL_VERSION, 7),
10615 )
10616 .unwrap();
10617 let ack = parse_ack(&responses[0]);
10618 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
10619
10620 let described = handler
10621 .handle_control_frame(
10622 &route_ctx(ConnectionId::new(2)).0,
10623 Frame::build(
10624 FrameType::Request,
10625 control_flags(),
10626 0,
10627 0,
10628 9,
10629 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
10630 )
10631 .unwrap(),
10632 )
10633 .await
10634 .unwrap();
10635 let ClientControlResponse::ServerDescribe { machine_id, .. } =
10636 serde_json::from_slice(&described[0].body).unwrap()
10637 else {
10638 panic!("server.describe answered with another shape");
10639 };
10640 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
10641 }
10642
10643 #[test]
10644 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
10645 let registry = Arc::new(Registry::default());
10648 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
10649 crate::daemon_config::StorageConfig::Sqlite {
10650 data_home: std::path::PathBuf::from("/data"),
10651 },
10652 ));
10653
10654 let responses = handler
10655 .handle_control(
10656 ConnectionId::new(1),
10657 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
10658 )
10659 .unwrap();
10660 let ack = parse_ack(&responses[0]);
10661 assert_eq!(
10662 ack.storage,
10663 Some(serde_json::json!({
10664 "module_id": "alfonso-routing",
10665 "storage_namespace": "default",
10666 "isolation": { "kind": "module" },
10667 "backend": {
10668 "backend": "sqlite",
10669 "path": "/data/cortexkit/alfonso-routing/store.db"
10670 }
10671 })),
10672 "the delivered descriptor is the module's own sqlite store path"
10673 );
10674 }
10675
10676 #[test]
10677 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
10678 let registry = Arc::new(Registry::default());
10679 let handler = ControlHandler::new(Arc::clone(®istry));
10680 let conn = ConnectionId::new(1);
10681 let responses = handler
10682 .handle_control(
10683 conn,
10684 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10685 )
10686 .unwrap();
10687 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
10688 let registration = registry.get_module("aft").unwrap().unwrap();
10689 assert_eq!(registration.control_ops, module_baseline_control_ops());
10690
10691 let frame =
10692 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
10693 assert!(handler
10694 .guard_module_control_op(&frame, "aft", "route.bind")
10695 .unwrap()
10696 .is_none());
10697 let error = handler
10698 .guard_module_control_op(&frame, "aft", "test.synthetic")
10699 .unwrap()
10700 .expect("synthetic ungranted op should be rejected");
10701 assert_eq!(error.header.ty, FrameType::Error);
10702 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
10703 }
10704
10705 #[test]
10706 fn hello_control_ops_some_adds_optional_grants() {
10707 let registry = Arc::new(Registry::default());
10708 let handler = ControlHandler::new(Arc::clone(®istry));
10709 handler
10710 .handle_control(
10711 ConnectionId::new(1),
10712 hello_frame_with_control_ops(
10713 "aft",
10714 PROTOCOL_VERSION,
10715 7,
10716 Some(vec![
10717 "future.synthetic".to_string(),
10718 "route.bind".to_string(),
10719 ]),
10720 ),
10721 )
10722 .unwrap();
10723 let registration = registry.get_module("aft").unwrap().unwrap();
10724 assert_eq!(
10725 registration.control_ops,
10726 vec![
10727 "route.bind".to_string(),
10728 "route.status".to_string(),
10729 "future.synthetic".to_string(),
10730 ]
10731 );
10732 let frame =
10733 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
10734 assert!(handler
10735 .guard_module_control_op(&frame, "aft", "future.synthetic")
10736 .unwrap()
10737 .is_none());
10738 }
10739
10740 #[tokio::test]
10741 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
10742 let registry = Arc::new(Registry::default());
10743 let forwarding = Arc::new(ForwardingTable::default());
10744 let handler =
10745 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10746 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
10747 hello_via_sink(
10748 &handler,
10749 &module_ctx,
10750 &mut module_rx,
10751 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
10752 )
10753 .await;
10754
10755 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
10756 let responses = handler
10757 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
10758 .await
10759 .unwrap();
10760 assert_eq!(responses.len(), 1);
10761 assert_eq!(responses[0].header.ty, FrameType::Error);
10762 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
10763 assert!(module_rx.try_recv().is_err());
10764 }
10765
10766 #[tokio::test]
10767 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
10768 let registry = Arc::new(Registry::default());
10769 let forwarding = Arc::new(ForwardingTable::default());
10770 let handler =
10771 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10772 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
10773 hello_via_sink(
10774 &handler,
10775 &module_ctx,
10776 &mut module_rx,
10777 hello_frame_with_control_ops(
10778 "aft",
10779 PROTOCOL_VERSION,
10780 7,
10781 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
10782 ),
10783 )
10784 .await;
10785
10786 let project_root = unique_project_root("demux");
10787 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
10788 let route_handler = handler.clone();
10789 let route_task = tokio::spawn(async move {
10790 route_handler
10791 .handle_control_frame(
10792 &route_client_ctx,
10793 route_open_frame(100, "aft", project_root),
10794 )
10795 .await
10796 .unwrap()
10797 });
10798 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10799 .await
10800 .unwrap()
10801 .unwrap();
10802 assert!(matches!(
10803 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
10804 ModuleControlRequest::RouteBind { .. }
10805 ));
10806
10807 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
10808 let health_handler = handler.clone();
10809 let health_task = tokio::spawn(async move {
10810 health_handler
10811 .handle_control_frame(
10812 &health_client_ctx,
10813 supervisor_health_probe_frame(101, "aft"),
10814 )
10815 .await
10816 .unwrap()
10817 });
10818 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
10819 .await
10820 .unwrap()
10821 .unwrap();
10822 assert_eq!(
10823 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
10824 ModuleControlRequest::HealthCheck {}
10825 );
10826
10827 handler
10828 .handle_control_frame(
10829 &module_ctx,
10830 health_response(health_frame.header.corr, HealthStatus::Degraded),
10831 )
10832 .await
10833 .unwrap();
10834 let health_response = health_task.await.unwrap();
10835 assert_eq!(health_response.len(), 1);
10836 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
10837 ClientControlResponse::SupervisorHealthProbe {
10838 module_id,
10839 status,
10840 detail,
10841 metrics,
10842 } => {
10843 assert_eq!(module_id, "aft");
10844 assert_eq!(status, HealthStatus::Degraded);
10845 assert_eq!(detail.as_deref(), Some("warming"));
10846 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
10847 }
10848 other => panic!("unexpected health response: {other:?}"),
10849 }
10850
10851 handler
10852 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
10853 .await
10854 .unwrap();
10855 let route_response = route_task.await.unwrap();
10856 assert!(route_response.is_empty());
10857 let published = route_client_rx.recv().await.unwrap();
10858 assert!(matches!(
10859 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
10860 ClientControlResponse::RouteOpen { .. }
10861 ));
10862 }
10863
10864 async fn relay_route_open(
10869 handler: &ControlHandler,
10870 client_connection: ConnectionId,
10871 client_egress: &FrameSink,
10872 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
10873 corr: u64,
10874 module_id: &str,
10875 project_root_label: &str,
10876 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
10877 let ctx = RouteCtx {
10878 connection_id: client_connection,
10879 egress: client_egress.clone(),
10880 };
10881 let handler = handler.clone();
10882 let project_root = unique_project_root(project_root_label);
10883 let module_id = module_id.to_string();
10884 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
10885 let task = tokio::spawn(async move {
10886 let _guard = tracing::dispatcher::set_default(&dispatch);
10887 handler
10888 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
10889 .await
10890 .unwrap()
10891 });
10892 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
10893 .await
10894 .expect("module receives the relayed route.bind")
10895 .expect("module egress is open");
10896 (task, bind.frame)
10897 }
10898
10899 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
10900 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
10901 ModuleControlRequest::RouteBind {
10902 route_channel,
10903 epoch,
10904 ..
10905 } => (route_channel, epoch),
10906 other => panic!("expected a route.bind request, got {other:?}"),
10907 }
10908 }
10909
10910 fn published_route(frame: &Frame) -> (u16, u32) {
10911 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
10912 ClientControlResponse::RouteOpen {
10913 route_channel,
10914 route_epoch,
10915 } => (route_channel, route_epoch),
10916 other => panic!("expected a route.open response, got {other:?}"),
10917 }
10918 }
10919
10920 #[tokio::test]
10940 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
10941 let registry = Arc::new(Registry::default());
10942 let forwarding = Arc::new(ForwardingTable::default());
10943 let handler =
10944 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
10945
10946 let module_connection = ConnectionId::new(30);
10947 let (module_ctx, mut module_rx) = route_ctx(module_connection);
10948 hello_via_sink(
10949 &handler,
10950 &module_ctx,
10951 &mut module_rx,
10952 hello_frame("aft", PROTOCOL_VERSION, 7),
10953 )
10954 .await;
10955
10956 let dying_client = ConnectionId::new(31);
10957 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
10958
10959 let (first_task, first_bind) = relay_route_open(
10962 &handler,
10963 dying_client,
10964 &dying_ctx.egress,
10965 &mut module_rx,
10966 100,
10967 "aft",
10968 "closing-first",
10969 )
10970 .await;
10971 handler
10972 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
10973 .await
10974 .unwrap();
10975 assert!(first_task.await.unwrap().is_empty());
10976 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
10977
10978 let (second_task, second_bind) = relay_route_open(
10980 &handler,
10981 dying_client,
10982 &dying_ctx.egress,
10983 &mut module_rx,
10984 101,
10985 "aft",
10986 "closing-second",
10987 )
10988 .await;
10989 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
10990
10991 assert!(forwarding
10994 .escalate_client_delivery_failure(
10995 dying_client,
10996 first_channel,
10997 first_epoch,
10998 CloseReason::new(
10999 "module_to_client_delivery_failed",
11000 "client egress refused a module frame",
11001 ),
11002 crate::forwarding::UndeliveredFrame {
11003 module_id: None,
11004 sink: &dying_ctx.egress,
11005 },
11006 )
11007 .unwrap());
11008 assert!(!dying_ctx.egress.is_closed());
11009
11010 let ack = handler
11012 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
11013 .await;
11014 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11015 if module_loop_error.is_some() {
11016 handler.cleanup_connection(module_connection).unwrap();
11020 }
11021 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11025 .await
11026 .ok()
11027 .flatten();
11028
11029 assert!(
11031 registry
11032 .get_module_by_connection(module_connection)
11033 .unwrap()
11034 .is_some(),
11035 "one client's closing connection ended the shared module connection: \
11036 {module_loop_error:?}"
11037 );
11038 let cotenant = ConnectionId::new(32);
11040 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11041 let (cotenant_task, cotenant_bind) = relay_route_open(
11042 &handler,
11043 cotenant,
11044 &cotenant_ctx.egress,
11045 &mut module_rx,
11046 102,
11047 "aft",
11048 "closing-cotenant",
11049 )
11050 .await;
11051 handler
11052 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
11053 .await
11054 .unwrap();
11055 assert!(cotenant_task.await.unwrap().is_empty());
11056 let (cotenant_channel, cotenant_epoch) =
11057 published_route(&cotenant_rx.recv().await.unwrap());
11058 assert!(matches!(
11059 forwarding
11060 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11061 .unwrap(),
11062 DataRoute::Client(DataRouteState::Bound(_))
11063 ));
11064
11065 let goodbye = post_ack_module_frame
11068 .expect("module receives a GOODBYE for the abandoned route channel");
11069 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11070 assert_eq!(goodbye.header.channel, abandoned_channel);
11071 assert_eq!(goodbye.header.epoch, abandoned_epoch);
11072
11073 assert!(dying_rx.try_recv().is_err());
11077 let second_response = second_task.await.unwrap();
11078 assert_eq!(second_response.len(), 1);
11079 assert_eq!(
11080 parse_error(&second_response[0])["code"],
11081 "target_unavailable"
11082 );
11083 }
11084
11085 async fn catalog_snapshot(
11088 handler: &ControlHandler,
11089 client_ctx: &RouteCtx,
11090 corr: u64,
11091 ) -> (u64, Vec<(String, bool)>) {
11092 let body =
11093 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
11094 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
11095 let reply = handler
11096 .handle_control_frame(client_ctx, request)
11097 .await
11098 .unwrap()
11099 .pop()
11100 .unwrap();
11101 let ClientControlResponse::CatalogList {
11102 generation,
11103 modules,
11104 ..
11105 } = serde_json::from_slice(&reply.body).unwrap()
11106 else {
11107 panic!("not a catalog.list reply");
11108 };
11109 (
11110 generation,
11111 modules
11112 .into_iter()
11113 .map(|entry| (entry.module_id, entry.ready))
11114 .collect(),
11115 )
11116 }
11117
11118 fn hello_held_before_forwarding() -> (
11124 ControlHandler,
11125 Arc<std::sync::Barrier>,
11126 std::thread::JoinHandle<Vec<Frame>>,
11127 RouteCtx,
11128 mpsc::Receiver<crate::router::OutboundFrame>,
11129 ) {
11130 let hold = Arc::new(std::sync::Barrier::new(2));
11131 let handler = ControlHandler::with_forwarding(
11132 Arc::new(Registry::default()),
11133 Arc::new(ForwardingTable::default()),
11134 )
11135 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
11136 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11137 let hello = {
11138 let handler = handler.clone();
11139 let module_ctx = module_ctx.clone();
11140 std::thread::spawn(move || {
11141 handler
11142 .handle_hello(
11143 module_ctx.connection_id,
11144 Some(module_ctx.egress.clone()),
11145 hello_frame("aft", PROTOCOL_VERSION, 7),
11146 )
11147 .unwrap()
11148 })
11149 };
11150 (handler, hold, hello, module_ctx, module_rx)
11151 }
11152
11153 #[tokio::test]
11162 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
11163 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
11164 let client = ConnectionId::new(41);
11165 let (client_ctx, mut client_rx) = route_ctx(client);
11166
11167 hold.wait();
11168 assert!(handler.registry.get_module("aft").unwrap().is_some());
11170 assert!(!handler
11171 .forwarding
11172 .has_live_module_connection("aft")
11173 .unwrap());
11174 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
11175 assert!(
11176 listed.is_empty(),
11177 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
11178 );
11179 let refused = handler
11180 .handle_control_frame(
11181 &client_ctx,
11182 route_open_frame(301, "aft", unique_project_root("listed-window")),
11183 )
11184 .await
11185 .unwrap();
11186 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
11187 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
11188 let duplicate = handler
11189 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
11190 .await
11191 .unwrap();
11192 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11193
11194 hold.wait();
11195 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11196 let ack = module_rx.try_recv().unwrap().frame;
11197 assert_eq!(ack.header.ty, FrameType::HelloAck);
11198 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11199 assert_eq!(listed, vec![("aft".to_string(), true)]);
11200
11201 let (open_task, bind) = relay_route_open(
11202 &handler,
11203 client,
11204 &client_ctx.egress,
11205 &mut module_rx,
11206 303,
11207 "aft",
11208 "listed-after",
11209 )
11210 .await;
11211 assert!(handler
11212 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11213 .await
11214 .unwrap()
11215 .is_empty());
11216 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
11217 published_route(&client_rx.try_recv().unwrap().frame);
11218 }
11219
11220 #[tokio::test]
11227 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11228 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11229 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11230 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11231
11232 hold.wait();
11233 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11234 assert!(listed.is_empty());
11235 assert_eq!(
11236 during, before,
11237 "the generation moved while the module was not listed"
11238 );
11239
11240 hold.wait();
11241 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11242 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11243 assert_eq!(listed, vec![("aft".to_string(), true)]);
11244 assert_eq!(
11245 after,
11246 before + 1,
11247 "the module appeared without exactly one generation bump"
11248 );
11249 }
11250
11251 #[tokio::test]
11256 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11257 let registry = Arc::new(Registry::default());
11258 let handler = ControlHandler::with_forwarding(
11259 Arc::clone(®istry),
11260 Arc::new(ForwardingTable::default()),
11261 );
11262 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11263 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11264 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11265 drop(module_rx);
11266
11267 let replies = handler
11268 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
11269 .await
11270 .unwrap();
11271 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
11272 assert!(registry.get_module("aft").unwrap().is_none());
11273 assert!(registry
11274 .get_module_by_connection(module_ctx.connection_id)
11275 .unwrap()
11276 .is_none());
11277 assert_eq!(
11278 catalog_snapshot(&handler, &client_ctx, 301).await,
11279 (before, Vec::new())
11280 );
11281 }
11282
11283 async fn route_open_held_after_bind_send() -> (
11287 ControlHandler,
11288 Arc<tokio::sync::Barrier>,
11289 RouteCtx,
11290 mpsc::Receiver<crate::router::OutboundFrame>,
11291 ) {
11292 let hold = Arc::new(tokio::sync::Barrier::new(2));
11293 let handler = ControlHandler::with_forwarding(
11294 Arc::new(Registry::default()),
11295 Arc::new(ForwardingTable::default()),
11296 )
11297 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
11298 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
11299 hello_via_sink(
11300 &handler,
11301 &module_ctx,
11302 &mut module_rx,
11303 hello_frame("aft", PROTOCOL_VERSION, 7),
11304 )
11305 .await;
11306 (handler, hold, module_ctx, module_rx)
11307 }
11308
11309 #[tokio::test]
11316 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
11317 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11318 let client = ConnectionId::new(41);
11319 let (client_ctx, mut client_rx) = route_ctx(client);
11320
11321 let (open_task, bind) = relay_route_open(
11322 &handler,
11323 client,
11324 &client_ctx.egress,
11325 &mut module_rx,
11326 200,
11327 "aft",
11328 "acked-before-mark",
11329 )
11330 .await;
11331 let (module_channel, module_epoch) = route_bind_channel(&bind);
11332 hold.wait().await;
11334
11335 assert!(handler
11337 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11338 .await
11339 .unwrap()
11340 .is_empty());
11341 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
11342
11343 hold.wait().await;
11344 assert!(
11345 open_task.await.unwrap().is_empty(),
11346 "route.open answers accepted"
11347 );
11348
11349 match module_rx.try_recv() {
11352 Err(mpsc::error::TryRecvError::Empty) => {}
11353 Ok(outbound) => panic!(
11354 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
11355 outbound.frame.header.ty,
11356 outbound.frame.header.channel,
11357 outbound.frame.header.epoch
11358 ),
11359 Err(err) => panic!("module egress closed: {err}"),
11360 }
11361
11362 let router = Router::with_control_handler(Arc::new(handler.clone()));
11365 let request = Frame::build(
11366 FrameType::Request,
11367 Flags::new(true, Priority::Interactive, false),
11368 client_channel,
11369 client_epoch,
11370 201,
11371 b"first request".to_vec(),
11372 )
11373 .unwrap();
11374 router
11375 .route_for_connection(&client_ctx, request)
11376 .await
11377 .unwrap();
11378 let delivered = module_rx.try_recv().unwrap().frame;
11379 assert_eq!(delivered.header.ty, FrameType::Request);
11380 assert_eq!(
11381 (delivered.header.channel, delivered.header.epoch),
11382 (module_channel, module_epoch)
11383 );
11384 assert_eq!(delivered.body, b"first request");
11385 }
11386
11387 #[tokio::test]
11393 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
11394 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11395 let client = ConnectionId::new(42);
11396 let (client_ctx, _client_rx) = route_ctx(client);
11397
11398 let (open_task, bind) = relay_route_open(
11399 &handler,
11400 client,
11401 &client_ctx.egress,
11402 &mut module_rx,
11403 300,
11404 "aft",
11405 "released-before-mark",
11406 )
11407 .await;
11408 let (module_channel, module_epoch) = route_bind_channel(&bind);
11409 hold.wait().await;
11410
11411 handler.cleanup_connection(client).unwrap();
11412 assert!(
11413 module_rx.try_recv().is_err(),
11414 "client teardown must not GOODBYE a bind route.open is still queueing"
11415 );
11416
11417 hold.wait().await;
11418 let response = open_task.await.unwrap();
11419 assert_eq!(response.len(), 1);
11420 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11421
11422 let goodbye = module_rx
11423 .try_recv()
11424 .expect("module is told to drop the binding the daemon abandoned")
11425 .frame;
11426 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11427 assert_eq!(
11428 (goodbye.header.channel, goodbye.header.epoch),
11429 (module_channel, module_epoch)
11430 );
11431 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11432 }
11433
11434 #[tokio::test]
11439 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
11440 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
11441 let client = ConnectionId::new(43);
11442 let (client_ctx, client_rx) = route_ctx(client);
11443
11444 let (open_task, bind) = relay_route_open(
11445 &handler,
11446 client,
11447 &client_ctx.egress,
11448 &mut module_rx,
11449 400,
11450 "aft",
11451 "gone-client-before-mark",
11452 )
11453 .await;
11454 let (module_channel, module_epoch) = route_bind_channel(&bind);
11455 hold.wait().await;
11456
11457 drop(client_rx);
11458 assert!(client_ctx.egress.is_closed());
11459 handler
11460 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11461 .await
11462 .unwrap();
11463
11464 hold.wait().await;
11465 let response = open_task.await.unwrap();
11466 assert_eq!(response.len(), 1);
11467 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
11468
11469 let goodbye = module_rx
11470 .try_recv()
11471 .expect("module is told to drop the binding it acked")
11472 .frame;
11473 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11474 assert_eq!(
11475 (goodbye.header.channel, goodbye.header.epoch),
11476 (module_channel, module_epoch)
11477 );
11478 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
11479 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
11480 }
11481
11482 #[test]
11489 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
11490 let handler = ControlHandler::default();
11491 let module_connection = ConnectionId::new(30);
11492 let client_connection = ConnectionId::new(31);
11493
11494 handler
11495 .refuse_to_end_module_connection_for_a_client(
11496 module_connection,
11497 77,
11498 ForwardingError::ConnectionClosing {
11499 connection_id: client_connection,
11500 },
11501 )
11502 .expect("a closing client must never end the module connection");
11503
11504 assert!(matches!(
11505 handler.refuse_to_end_module_connection_for_a_client(
11506 module_connection,
11507 78,
11508 ForwardingError::ConnectionClosing {
11509 connection_id: module_connection,
11510 },
11511 ),
11512 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
11513 connection_id
11514 })) if connection_id == module_connection
11515 ));
11516 assert!(matches!(
11517 handler.refuse_to_end_module_connection_for_a_client(
11518 module_connection,
11519 79,
11520 ForwardingError::Poisoned,
11521 ),
11522 Err(RouterError::Forwarding(ForwardingError::Poisoned))
11523 ));
11524 assert!(matches!(
11525 handler.refuse_to_end_module_connection_for_a_client(
11526 module_connection,
11527 80,
11528 ForwardingError::StaleModuleEndpoint,
11529 ),
11530 Err(RouterError::Forwarding(
11531 ForwardingError::StaleModuleEndpoint
11532 ))
11533 ));
11534 }
11535
11536 #[tokio::test]
11549 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
11550 let registry = Arc::new(Registry::default());
11551 let forwarding = Arc::new(ForwardingTable::default());
11552 let supervisor = SupervisorHandle::new();
11553 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11554 let handler =
11555 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11556 .with_supervisor(supervisor);
11557
11558 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
11559 hello_via_sink(
11560 &handler,
11561 &target_ctx,
11562 &mut target_rx,
11563 hello_frame("target", PROTOCOL_VERSION, 1),
11564 )
11565 .await;
11566
11567 let wrong_nonce = handler
11571 .handle_control_frame(
11572 &route_ctx(ConnectionId::new(91)).0,
11573 route_open_frame_with_admission_facts(
11574 20,
11575 "target",
11576 unique_project_root("admission-facts"),
11577 Some(subc_control::ConsumerIdentity {
11578 module_id: "fed".to_string(),
11579 launch_nonce: "not-the-real-nonce".to_string(),
11580 }),
11581 None,
11582 ),
11583 )
11584 .await
11585 .unwrap();
11586 assert_eq!(
11587 parse_error(&wrong_nonce[0])["code"],
11588 "bad_consumer_identity",
11589 "a mismatched launch nonce must be refused, not stamped Reserved"
11590 );
11591
11592 let never_spawned = handler
11596 .handle_control_frame(
11597 &route_ctx(ConnectionId::new(92)).0,
11598 route_open_frame_with_admission_facts(
11599 21,
11600 "target",
11601 unique_project_root("admission-facts"),
11602 Some(subc_control::ConsumerIdentity {
11603 module_id: "never-spawned".to_string(),
11604 launch_nonce: "any-nonce".to_string(),
11605 }),
11606 None,
11607 ),
11608 )
11609 .await
11610 .unwrap();
11611 assert_eq!(
11612 parse_error(&never_spawned[0])["code"],
11613 "bad_consumer_identity",
11614 "an unspawned module_id must be refused rather than accepted for lack of a record"
11615 );
11616 }
11617
11618 #[tokio::test]
11641 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
11642 let registry = Arc::new(Registry::default());
11643 let forwarding = Arc::new(ForwardingTable::default());
11644 let supervisor = SupervisorHandle::new();
11645 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11646 let handler =
11647 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11648 .with_supervisor(supervisor);
11649
11650 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
11651 hello_via_sink(
11652 &handler,
11653 &target_ctx,
11654 &mut target_rx,
11655 hello_frame("target", PROTOCOL_VERSION, 1),
11656 )
11657 .await;
11658
11659 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
11660 let route_handler = handler.clone();
11661 let route_task = tokio::spawn(async move {
11662 route_handler
11663 .handle_control_frame(
11664 &client_ctx,
11665 route_open_frame_with_admission_facts(
11666 30,
11667 "target",
11668 unique_project_root("admission-facts"),
11669 Some(subc_control::ConsumerIdentity {
11670 module_id: "fed".to_string(),
11671 launch_nonce: "fed-nonce".to_string(),
11672 }),
11673 None,
11674 ),
11675 )
11676 .await
11677 .unwrap()
11678 });
11679
11680 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11687 .await
11688 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
11689 .expect("module control channel closed before route.bind");
11690 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
11691 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
11692 panic!("expected route.bind")
11693 };
11694 assert_eq!(
11695 principal,
11696 Some(Principal::Reserved {
11697 module_id: "fed".to_string()
11698 }),
11699 "a correctly attested consumer must be stamped Reserved for its own id"
11700 );
11701
11702 handler
11703 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
11704 .await
11705 .unwrap();
11706 assert!(route_task.await.unwrap().is_empty());
11707 assert!(
11708 matches!(
11709 serde_json::from_slice::<ClientControlResponse>(
11710 &client_rx.recv().await.unwrap().body
11711 )
11712 .unwrap(),
11713 ClientControlResponse::RouteOpen { .. }
11714 ),
11715 "the route must actually open, not merely avoid an error"
11716 );
11717 }
11718
11719 #[tokio::test(start_paused = true)]
11720 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
11721 let registry = Arc::new(Registry::default());
11722 let forwarding = Arc::new(ForwardingTable::default());
11723 let supervisor = SupervisorHandle::new();
11724 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
11725 let handler =
11726 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
11727 .with_supervisor(supervisor);
11728
11729 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
11730 hello_via_sink(
11731 &handler,
11732 &target_ctx,
11733 &mut target_rx,
11734 hello_frame("target", PROTOCOL_VERSION, 1),
11735 )
11736 .await;
11737
11738 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
11739 let direct_handler = handler.clone();
11740 let direct_open = tokio::spawn(async move {
11741 direct_handler
11742 .handle_control_frame(
11743 &direct_ctx,
11744 route_open_frame(2, "target", unique_project_root("route-census-direct")),
11745 )
11746 .await
11747 .unwrap()
11748 });
11749 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11750 .await
11751 .expect("no direct route.bind within 5s")
11752 .expect("target control channel closed before direct route.bind");
11753 handler
11754 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
11755 .await
11756 .unwrap();
11757 assert!(direct_open.await.unwrap().is_empty());
11758 let _ = direct_rx.recv().await.unwrap();
11759
11760 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
11761 let reserved_handler = handler.clone();
11762 let reserved_open = tokio::spawn(async move {
11763 reserved_handler
11764 .handle_control_frame(
11765 &reserved_ctx,
11766 route_open_frame_with_admission_facts(
11767 3,
11768 "target",
11769 unique_project_root("admission-facts"),
11770 Some(ConsumerIdentity {
11771 module_id: "fed".to_string(),
11772 launch_nonce: "fed-nonce".to_string(),
11773 }),
11774 None,
11775 ),
11776 )
11777 .await
11778 .unwrap()
11779 });
11780 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11781 .await
11782 .expect("no reserved route.bind within 5s")
11783 .expect("target control channel closed before reserved route.bind");
11784 handler
11785 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
11786 .await
11787 .unwrap();
11788 assert!(reserved_open.await.unwrap().is_empty());
11789 let _ = reserved_rx.recv().await.unwrap();
11790
11791 forwarding
11792 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
11793 .unwrap();
11794 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
11795 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
11796 module_id: Some("target".to_string()),
11797 })
11798 .unwrap();
11799 let census_frame =
11800 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
11801 let response = handler
11802 .handle_control_frame(&census_ctx, census_frame)
11803 .await
11804 .unwrap()
11805 .pop()
11806 .unwrap();
11807 let actual: Value = serde_json::from_slice(&response.body).unwrap();
11808 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
11809 assert!(matches!(
11810 decoded,
11811 ClientControlResponse::SupervisorRoutes { .. }
11812 ));
11813 let routes = actual["modules"][0]["routes"].as_array().unwrap();
11814 assert_eq!(routes.len(), 2);
11815 assert!(routes.iter().all(|route| route["draining"] == true));
11816 assert!(
11819 routes.iter().all(|route| route["drain_reason"] == "reload"),
11820 "draining routes must name the drain's reason: {routes:?}"
11821 );
11822 assert!(routes.iter().any(|route| {
11823 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
11824 }));
11825 assert!(routes.iter().any(|route| {
11826 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
11827 }));
11828
11829 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
11830 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
11831 if std::env::var_os("UPDATE_GOLDEN").is_some() {
11832 std::fs::write(
11833 &golden_path,
11834 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
11835 )
11836 .unwrap();
11837 }
11838 let expected: Value =
11839 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
11840 assert_eq!(actual, expected);
11841 }
11842
11843 async fn query_live_roots(
11844 handler: &ControlHandler,
11845 module_ctx: &RouteCtx,
11846 ) -> ModuleControlResponseToModule {
11847 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
11848 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
11849 let response = handler
11850 .handle_control_frame(module_ctx, frame)
11851 .await
11852 .unwrap()
11853 .pop()
11854 .unwrap();
11855 serde_json::from_slice(&response.body).unwrap()
11856 }
11857
11858 #[tokio::test(start_paused = true)]
11859 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
11860 let registry = Arc::new(Registry::default());
11861 let forwarding = Arc::new(ForwardingTable::default());
11862 let handler = ControlHandler::with_forwarding(registry, forwarding);
11863 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
11864 hello_via_sink(
11865 &handler,
11866 &target_ctx,
11867 &mut target_rx,
11868 hello_frame("target", PROTOCOL_VERSION, 1),
11869 )
11870 .await;
11871 let root = unique_project_root("live-roots-known");
11872 let path = ProjectRootId::from_path_allowing_missing(root.path())
11873 .unwrap()
11874 .as_path()
11875 .to_path_buf();
11876 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
11877 let open_handler = handler.clone();
11878 let opened = tokio::spawn(async move {
11879 open_handler
11880 .handle_control_frame(&client_ctx, route_open_frame(2, "target", root))
11881 .await
11882 .unwrap()
11883 });
11884 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11885 .await
11886 .unwrap()
11887 .unwrap();
11888 handler
11889 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
11890 .await
11891 .unwrap();
11892 assert!(opened.await.unwrap().is_empty());
11893 let _ = client_rx.recv().await.unwrap();
11894
11895 let root = unique_project_root("live-roots-pending");
11896 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
11897 .unwrap()
11898 .as_path()
11899 .to_path_buf();
11900 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
11901 let open_handler = handler.clone();
11902 let pending = tokio::spawn(async move {
11903 open_handler
11904 .handle_control_frame(&client_ctx, route_open_frame(3, "target", root))
11905 .await
11906 .unwrap()
11907 });
11908 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
11909 .await
11910 .unwrap()
11911 .unwrap();
11912 let actual = query_live_roots(&handler, &target_ctx).await;
11913 let ModuleControlResponseToModule::LiveRoots {
11914 roots,
11915 unknown_root_bindings,
11916 total_bindings,
11917 } = actual
11918 else {
11919 panic!("expected live roots")
11920 };
11921 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
11922 assert_eq!(unknown_root_bindings, 0);
11923 assert_eq!(
11924 roots.len(),
11925 2,
11926 "root-known arm must retain each canonical root"
11927 );
11928 assert_eq!(
11929 total_bindings,
11930 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11931 );
11932 let counts = roots
11933 .iter()
11934 .map(|root| (root.project_root.clone(), root.bound, root.pending))
11935 .collect::<Vec<_>>();
11936 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
11937 expected.sort_by(|a, b| a.0.cmp(&b.0));
11938 assert_eq!(
11939 counts, expected,
11940 "roots must sort by path and count pending separately"
11941 );
11942 handler
11943 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
11944 .await
11945 .unwrap();
11946 assert!(pending.await.unwrap().is_empty());
11947 }
11948
11949 #[tokio::test(start_paused = true)]
11950 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
11951 let forwarding = Arc::new(ForwardingTable::default());
11952 let handler =
11953 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
11954 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
11955 hello_via_sink(
11956 &handler,
11957 &target_ctx,
11958 &mut target_rx,
11959 hello_frame("target", PROTOCOL_VERSION, 1),
11960 )
11961 .await;
11962 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
11963 let pending = forwarding
11964 .begin_route_bind_relay_for_test(
11965 client_ctx.connection_id,
11966 client_ctx.egress.clone(),
11967 2,
11968 "target",
11969 )
11970 .unwrap();
11971 forwarding
11972 .complete_pending_relay(
11973 target_ctx.connection_id,
11974 pending.corr,
11975 RouteBindRelayOutcome::Accepted,
11976 )
11977 .unwrap();
11978 let actual = query_live_roots(&handler, &target_ctx).await;
11979 let ModuleControlResponseToModule::LiveRoots {
11980 roots,
11981 unknown_root_bindings,
11982 total_bindings,
11983 } = actual
11984 else {
11985 panic!("expected live roots")
11986 };
11987 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
11988 assert_eq!(
11989 unknown_root_bindings, 1,
11990 "unknown-root arm must not read as no bindings"
11991 );
11992 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
11993 assert_eq!(
11994 total_bindings,
11995 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
11996 );
11997 }
11998
11999 #[tokio::test(start_paused = true)]
12007 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
12008 let forwarding = Arc::new(ForwardingTable::default());
12009 let handler =
12010 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12011 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
12012 let replies = handler
12013 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
12014 .await
12015 .unwrap();
12016 let queued_by_hello = module_rx.len();
12017
12018 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
12019 let open_handler = handler.clone();
12020 let open = tokio::spawn(async move {
12021 open_handler
12022 .handle_control_frame(
12023 &client_ctx,
12024 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
12025 )
12026 .await
12027 .unwrap()
12028 });
12029 let mut spins = 0;
12031 while module_rx.len() == queued_by_hello {
12032 spins += 1;
12033 assert!(spins < 10_000, "route.open never queued a route.bind");
12034 tokio::task::yield_now().await;
12035 }
12036
12037 for reply in replies {
12039 module_ctx.egress.send(reply).await.unwrap();
12040 }
12041
12042 let first = module_rx.recv().await.unwrap().frame;
12043 assert_eq!(
12044 first.header.ty,
12045 FrameType::HelloAck,
12046 "the first frame a registering module reads must be its HELLO_ACK"
12047 );
12048 assert_eq!(first.header.corr, 7);
12049 let second = module_rx.recv().await.unwrap().frame;
12050 assert_eq!(second.header.ty, FrameType::Request);
12051 assert!(
12052 matches!(
12053 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
12054 ModuleControlRequest::RouteBind { .. }
12055 ),
12056 "the route.bind follows the ack"
12057 );
12058 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
12059
12060 handler
12061 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
12062 .await
12063 .unwrap();
12064 assert!(open.await.unwrap().is_empty());
12065 let _ = client_rx.recv().await.unwrap();
12066 }
12067
12068 #[tokio::test(start_paused = true)]
12069 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
12070 let handler = ControlHandler::with_forwarding(
12071 Arc::new(Registry::default()),
12072 Arc::new(ForwardingTable::default()),
12073 );
12074 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
12075 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
12076 hello_via_sink(
12077 &handler,
12078 &first_ctx,
12079 &mut first_rx,
12080 hello_frame("first", PROTOCOL_VERSION, 1),
12081 )
12082 .await;
12083 hello_via_sink(
12084 &handler,
12085 &second_ctx,
12086 &mut second_rx,
12087 hello_frame("second", PROTOCOL_VERSION, 2),
12088 )
12089 .await;
12090 let root = unique_project_root("second-only");
12091 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
12092 let cloned = handler.clone();
12093 let open = tokio::spawn(async move {
12094 cloned
12095 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
12096 .await
12097 .unwrap()
12098 });
12099 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
12100 .await
12101 .unwrap()
12102 .unwrap();
12103 let first = query_live_roots(&handler, &first_ctx).await;
12104 let second = query_live_roots(&handler, &second_ctx).await;
12105 assert!(
12106 matches!(
12107 first,
12108 ModuleControlResponseToModule::LiveRoots {
12109 total_bindings: 0,
12110 ..
12111 }
12112 ),
12113 "cross-module scope must not expose another module's roots"
12114 );
12115 assert!(
12116 matches!(
12117 second,
12118 ModuleControlResponseToModule::LiveRoots {
12119 total_bindings: 1,
12120 ..
12121 }
12122 ),
12123 "second module must see its pending route"
12124 );
12125 handler
12126 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
12127 .await
12128 .unwrap();
12129 assert!(open.await.unwrap().is_empty());
12130 }
12131
12132 #[tokio::test(start_paused = true)]
12133 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
12134 let handler = ControlHandler::with_forwarding(
12135 Arc::new(Registry::default()),
12136 Arc::new(ForwardingTable::default()),
12137 );
12138 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
12139 hello_via_sink(
12140 &handler,
12141 &target_ctx,
12142 &mut target_rx,
12143 hello_frame("target", PROTOCOL_VERSION, 1),
12144 )
12145 .await;
12146 let actual = query_live_roots(&handler, &target_ctx).await;
12147 let ModuleControlResponseToModule::LiveRoots {
12148 roots,
12149 unknown_root_bindings,
12150 total_bindings,
12151 } = actual
12152 else {
12153 panic!("expected live roots")
12154 };
12155 assert!(roots.is_empty());
12156 assert_eq!(unknown_root_bindings, 0);
12157 assert_eq!(total_bindings, 0);
12158 assert_eq!(
12159 total_bindings,
12160 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12161 );
12162 }
12163
12164 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
12171 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12172 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
12173 let text = std::fs::read_to_string(&path)
12174 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
12175 let vectors: Vec<(String, Value)> = text
12176 .lines()
12177 .filter(|line| !line.trim().is_empty())
12178 .map(|line| {
12179 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
12180 let id = entry["corpus_id"]
12181 .as_str()
12182 .expect("every vector carries a corpus_id")
12183 .to_string();
12184 (id, entry["package"].clone())
12185 })
12186 .collect();
12187 assert_eq!(
12191 vectors.len(),
12192 3,
12193 "vendored fed corpus changed size; re-sync from subc-federation"
12194 );
12195
12196 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
12211 let richest = vectors
12212 .iter()
12213 .filter_map(|(_, package)| package.as_object())
12214 .map(|object| {
12215 object
12216 .keys()
12217 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
12218 .count()
12219 })
12220 .max()
12221 .unwrap_or(0);
12222 assert!(
12223 richest >= 2,
12224 "vendored corpus no longer carries a package with unmodelled fields, \
12225 so the relay test can no longer distinguish a verbatim relay from a lossy one"
12226 );
12227
12228 vectors
12229 }
12230
12231 #[tokio::test]
12239 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12240 for (corpus_id, package) in fed_admission_facts_vectors() {
12241 let registry = Arc::new(Registry::default());
12242 let forwarding = Arc::new(ForwardingTable::default());
12243 let supervisor = SupervisorHandle::new();
12244 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12245 let handler =
12246 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12247 .with_supervisor(supervisor)
12248 .with_admission_facts_config(
12249 Some("fed".to_string()),
12250 Some(vec!["target".to_string()]),
12251 );
12252
12253 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12254 hello_via_sink(
12255 &handler,
12256 &target_ctx,
12257 &mut target_rx,
12258 hello_frame("target", PROTOCOL_VERSION, 1),
12259 )
12260 .await;
12261
12262 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
12263 let route_handler = handler.clone();
12264 let expected = package.clone();
12265 let route_task = tokio::spawn(async move {
12266 route_handler
12267 .handle_control_frame(
12268 &client_ctx,
12269 route_open_frame_with_admission_facts(
12270 20,
12271 "target",
12272 unique_project_root("admission-facts"),
12273 Some(subc_control::ConsumerIdentity {
12274 module_id: "fed".to_string(),
12275 launch_nonce: "fed-nonce".to_string(),
12276 }),
12277 Some(package),
12278 ),
12279 )
12280 .await
12281 .unwrap()
12282 });
12283
12284 let bind_frame = target_rx.recv().await.unwrap();
12285 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12286 let ModuleControlRequest::RouteBind {
12287 admission_facts, ..
12288 } = bind
12289 else {
12290 panic!("{corpus_id}: expected route.bind")
12291 };
12292 assert_eq!(
12293 admission_facts,
12294 Some(expected),
12295 "{corpus_id}: relay must not add, drop or reshape any field"
12296 );
12297
12298 handler
12299 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12300 .await
12301 .unwrap();
12302 route_task.await.unwrap();
12303 }
12304 }
12305
12306 #[tokio::test]
12307 async fn route_bind_never_relays_the_openers_build_class() {
12308 let registry = Arc::new(Registry::default());
12309 let forwarding = Arc::new(ForwardingTable::default());
12310 let handler =
12311 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12312 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
12313 hello_via_sink(
12314 &handler,
12315 &target_ctx,
12316 &mut target_rx,
12317 hello_frame("target", PROTOCOL_VERSION, 1),
12318 )
12319 .await;
12320
12321 let root = unique_project_root("build-class");
12322 let mut identity = BindIdentity::new(
12323 root.path().to_path_buf(),
12324 "unit".to_string(),
12325 "s".to_string(),
12326 );
12327 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
12328 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
12329 target: RouteTarget::ToolProvider {
12330 module_id: "target".to_string(),
12331 },
12332 identity,
12333 consumer_identity: None,
12334 consumer_capabilities: None,
12335 role_versions: None,
12336 admission_facts: None,
12337 scope: None,
12338 })
12339 .unwrap();
12340 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
12341
12342 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
12343 let route_handler = handler.clone();
12344 let route_task = tokio::spawn(async move {
12345 route_handler
12346 .handle_control_frame(&client_ctx, open)
12347 .await
12348 .unwrap()
12349 });
12350 let bind_frame = target_rx.recv().await.unwrap();
12351 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12352 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
12353 panic!("expected route.bind")
12354 };
12355 assert_eq!(
12356 identity.build_class, None,
12357 "route.bind carried the build class"
12358 );
12359 assert!(
12360 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
12361 "the route.bind bytes still name build_class"
12362 );
12363 handler
12364 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12365 .await
12366 .unwrap();
12367 route_task.await.unwrap();
12368 }
12369
12370 #[tokio::test]
12371 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
12372 let registry = Arc::new(Registry::default());
12373 let forwarding = Arc::new(ForwardingTable::default());
12374 let supervisor = SupervisorHandle::new();
12375 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12376 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
12377 let handler =
12378 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12379 .with_supervisor(supervisor)
12380 .with_admission_facts_config(
12381 Some("fed".to_string()),
12382 Some(vec!["target".to_string()]),
12383 );
12384
12385 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
12386 hello_via_sink(
12387 &handler,
12388 &target_ctx,
12389 &mut target_rx,
12390 hello_frame("target", PROTOCOL_VERSION, 1),
12391 )
12392 .await;
12393 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
12394 hello_via_sink(
12395 &handler,
12396 &other_ctx,
12397 &mut other_rx,
12398 hello_frame("other", PROTOCOL_VERSION, 2),
12399 )
12400 .await;
12401
12402 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
12403 let expected_facts = facts.clone();
12404 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
12405 let route_handler = handler.clone();
12406 let route_task = tokio::spawn(async move {
12407 route_handler
12408 .handle_control_frame(
12409 &client_ctx,
12410 route_open_frame_with_admission_facts(
12411 10,
12412 "target",
12413 unique_project_root("admission-facts"),
12414 Some(subc_control::ConsumerIdentity {
12415 module_id: "fed".to_string(),
12416 launch_nonce: "fed-nonce".to_string(),
12417 }),
12418 Some(facts.clone()),
12419 ),
12420 )
12421 .await
12422 .unwrap()
12423 });
12424 let bind_frame = target_rx.recv().await.unwrap();
12425 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12426 let ModuleControlRequest::RouteBind {
12427 admission_facts, ..
12428 } = bind
12429 else {
12430 panic!("expected route.bind")
12431 };
12432 assert_eq!(admission_facts, Some(expected_facts));
12433 handler
12434 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12435 .await
12436 .unwrap();
12437 assert!(route_task.await.unwrap().is_empty());
12438 assert!(matches!(
12439 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
12440 .unwrap(),
12441 ClientControlResponse::RouteOpen { .. }
12442 ));
12443
12444 let direct = handler
12445 .handle_control_frame(
12446 &route_ctx(ConnectionId::new(73)).0,
12447 route_open_frame_with_admission_facts(
12448 11,
12449 "target",
12450 unique_project_root("admission-facts"),
12451 None,
12452 Some(json!({"x": 1})),
12453 ),
12454 )
12455 .await
12456 .unwrap();
12457 assert_eq!(
12458 parse_error(&direct[0])["code"],
12459 "admission_facts_not_permitted"
12460 );
12461
12462 let different_reserved = handler
12463 .handle_control_frame(
12464 &route_ctx(ConnectionId::new(77)).0,
12465 route_open_frame_with_admission_facts(
12466 15,
12467 "target",
12468 unique_project_root("admission-facts"),
12469 Some(subc_control::ConsumerIdentity {
12470 module_id: "other".to_string(),
12471 launch_nonce: "other-nonce".to_string(),
12472 }),
12473 Some(json!({"x": 1})),
12474 ),
12475 )
12476 .await
12477 .unwrap();
12478 assert_eq!(
12479 parse_error(&different_reserved[0])["code"],
12480 "admission_facts_not_permitted"
12481 );
12482
12483 let other_target = handler
12484 .handle_control_frame(
12485 &route_ctx(ConnectionId::new(74)).0,
12486 route_open_frame_with_admission_facts(
12487 12,
12488 "other",
12489 unique_project_root("admission-facts"),
12490 Some(subc_control::ConsumerIdentity {
12491 module_id: "fed".to_string(),
12492 launch_nonce: "fed-nonce".to_string(),
12493 }),
12494 Some(json!({"x": 1})),
12495 ),
12496 )
12497 .await
12498 .unwrap();
12499 assert_eq!(
12500 parse_error(&other_target[0])["code"],
12501 "admission_facts_target_not_allowed"
12502 );
12503
12504 let nonexistent = handler
12505 .handle_control_frame(
12506 &route_ctx(ConnectionId::new(75)).0,
12507 route_open_frame_with_admission_facts(
12508 13,
12509 "missing",
12510 unique_project_root("admission-facts"),
12511 None,
12512 Some(json!({"x": 1})),
12513 ),
12514 )
12515 .await
12516 .unwrap();
12517 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
12518
12519 let described = handler
12520 .handle_control_frame(
12521 &route_ctx(ConnectionId::new(76)).0,
12522 Frame::build(
12523 FrameType::Request,
12524 control_flags(),
12525 0,
12526 0,
12527 14,
12528 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
12529 )
12530 .unwrap(),
12531 )
12532 .await
12533 .unwrap();
12534 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12535 serde_json::from_slice(&described[0].body).unwrap()
12536 else {
12537 panic!("expected server.describe response")
12538 };
12539 assert!(capabilities
12540 .iter()
12541 .any(|cap| cap == "admission_facts_relay_v1"));
12542 }
12543
12544 #[tokio::test]
12545 async fn admission_facts_without_configured_carrier_are_rejected() {
12546 let registry = Arc::new(Registry::default());
12547 let forwarding = Arc::new(ForwardingTable::default());
12548 let handler = ControlHandler::with_forwarding(registry, forwarding);
12549 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
12550 hello_via_sink(
12551 &handler,
12552 &target_ctx,
12553 &mut target_rx,
12554 hello_frame("target", PROTOCOL_VERSION, 1),
12555 )
12556 .await;
12557
12558 let responses = handler
12559 .handle_control_frame(
12560 &route_ctx(ConnectionId::new(79)).0,
12561 route_open_frame_with_admission_facts(
12562 16,
12563 "target",
12564 unique_project_root("admission-facts"),
12565 None,
12566 Some(json!({"x": 1})),
12567 ),
12568 )
12569 .await
12570 .unwrap();
12571 assert_eq!(
12572 parse_error(&responses[0])["code"],
12573 "admission_facts_not_permitted"
12574 );
12575 }
12576
12577 #[tokio::test]
12578 async fn route_open_relays_consumer_capabilities_verbatim() {
12579 let registry = Arc::new(Registry::default());
12580 let forwarding = Arc::new(ForwardingTable::default());
12581 let handler =
12582 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12583 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
12584 hello_via_sink(
12585 &handler,
12586 &module_ctx,
12587 &mut module_rx,
12588 hello_frame("aft", PROTOCOL_VERSION, 7),
12589 )
12590 .await;
12591
12592 let expected = vec!["elicitation".to_string(), "roots".to_string()];
12593 let expected_for_request = expected.clone();
12594 let project_root = unique_project_root("consumer-capabilities-present");
12595 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
12596 let route_handler = handler.clone();
12597 let route_task = tokio::spawn(async move {
12598 route_handler
12599 .handle_control_frame(
12600 &client_ctx,
12601 route_open_frame_with_consumer_capabilities(
12602 401,
12603 "aft",
12604 project_root,
12605 Some(expected_for_request),
12606 ),
12607 )
12608 .await
12609 .unwrap()
12610 });
12611 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12612 .await
12613 .unwrap()
12614 .unwrap();
12615 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12616 let ModuleControlRequest::RouteBind {
12617 consumer_capabilities,
12618 ..
12619 } = bind
12620 else {
12621 panic!("expected route.bind request, got {bind:?}");
12622 };
12623 assert_eq!(consumer_capabilities, Some(expected.clone()));
12624
12625 handler
12626 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12627 .await
12628 .unwrap();
12629 let route_response = route_task.await.unwrap();
12630 assert!(route_response.is_empty());
12631 let published = client_rx.recv().await.unwrap();
12632 assert!(matches!(
12633 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12634 ClientControlResponse::RouteOpen { .. }
12635 ));
12636 }
12637
12638 #[tokio::test]
12639 async fn route_open_without_consumer_capabilities_relays_none() {
12640 let registry = Arc::new(Registry::default());
12641 let forwarding = Arc::new(ForwardingTable::default());
12642 let handler =
12643 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12644 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
12645 hello_via_sink(
12646 &handler,
12647 &module_ctx,
12648 &mut module_rx,
12649 hello_frame("aft", PROTOCOL_VERSION, 7),
12650 )
12651 .await;
12652
12653 let project_root = unique_project_root("consumer-capabilities-absent");
12654 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
12655 let route_handler = handler.clone();
12656 let route_task = tokio::spawn(async move {
12657 route_handler
12658 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
12659 .await
12660 .unwrap()
12661 });
12662 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12663 .await
12664 .unwrap()
12665 .unwrap();
12666 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12667 let ModuleControlRequest::RouteBind {
12668 consumer_capabilities,
12669 ..
12670 } = bind
12671 else {
12672 panic!("expected route.bind request, got {bind:?}");
12673 };
12674 assert_eq!(consumer_capabilities, None);
12675
12676 handler
12677 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12678 .await
12679 .unwrap();
12680 let route_response = route_task.await.unwrap();
12681 assert!(route_response.is_empty());
12682 let published = client_rx.recv().await.unwrap();
12683 assert!(matches!(
12684 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12685 ClientControlResponse::RouteOpen { .. }
12686 ));
12687 }
12688
12689 async fn bind_role_versions_for(
12693 sent: Option<BTreeMap<String, String>>,
12694 connection: u64,
12695 ) -> Option<BTreeMap<String, String>> {
12696 let registry = Arc::new(Registry::default());
12697 let forwarding = Arc::new(ForwardingTable::default());
12698 let handler =
12699 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12700 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
12701 hello_via_sink(
12702 &handler,
12703 &module_ctx,
12704 &mut module_rx,
12705 hello_frame("aft", PROTOCOL_VERSION, 7),
12706 )
12707 .await;
12708 let project_root = unique_project_root("role-versions");
12709 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
12710 let route_handler = handler.clone();
12711 let route_task = tokio::spawn(async move {
12712 route_handler
12713 .handle_control_frame(
12714 &client_ctx,
12715 route_open_frame_with_role_versions(403, "aft", project_root, sent),
12716 )
12717 .await
12718 .unwrap()
12719 });
12720 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
12721 .await
12722 .expect("a well-formed route.open reaches the module as a bind")
12723 .unwrap();
12724 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12725 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
12726 panic!("expected route.bind request, got {bind:?}");
12727 };
12728 handler
12729 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
12730 .await
12731 .unwrap();
12732 assert!(route_task.await.unwrap().is_empty());
12733 let published = client_rx.recv().await.unwrap();
12734 assert!(matches!(
12735 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
12736 ClientControlResponse::RouteOpen { .. }
12737 ));
12738 role_versions
12739 }
12740
12741 #[tokio::test]
12742 async fn route_open_relays_role_versions_verbatim() {
12743 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
12744 assert_eq!(
12745 bind_role_versions_for(Some(sent.clone()), 141).await,
12746 Some(sent)
12747 );
12748 }
12749
12750 #[tokio::test]
12753 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
12754 assert_eq!(bind_role_versions_for(None, 143).await, None);
12755 assert_eq!(
12756 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
12757 None
12758 );
12759 }
12760
12761 #[tokio::test]
12762 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
12763 let registry = Arc::new(Registry::default());
12764 let forwarding = Arc::new(ForwardingTable::default());
12765 let handler =
12766 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
12767 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
12768 hello_via_sink(
12769 &handler,
12770 &module_ctx,
12771 &mut module_rx,
12772 hello_frame("aft", PROTOCOL_VERSION, 7),
12773 )
12774 .await;
12775
12776 let nine: BTreeMap<String, String> = (0..9)
12777 .map(|index| (format!("role-{index}"), "v1".to_string()))
12778 .collect();
12779 for (label, malformed) in [
12780 (
12781 "invalid role name",
12782 role_versions(&[("Tool_Provider", "v1")]),
12783 ),
12784 ("invalid version", role_versions(&[("tool-provider", "v0")])),
12785 ("nine entries", nine),
12786 ] {
12787 let responses = tokio::time::timeout(
12790 Duration::from_secs(1),
12791 handler.handle_control_frame(
12792 &route_ctx(ConnectionId::new(148)).0,
12793 route_open_frame_with_role_versions(
12794 404,
12795 "aft",
12796 unique_project_root("role-versions-malformed"),
12797 Some(malformed),
12798 ),
12799 ),
12800 )
12801 .await
12802 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
12803 .unwrap();
12804 assert_eq!(responses.len(), 1, "{label}");
12805 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
12806 let error = parse_error(&responses[0]);
12807 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
12808 assert_eq!(
12809 error["detail"]["field"], "role_versions",
12810 "{label}: {error}"
12811 );
12812 assert!(
12813 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
12814 "{label}: a malformed declaration is terminal"
12815 );
12816 assert!(
12817 module_rx.try_recv().is_err(),
12818 "{label}: the module must never see a bind"
12819 );
12820 }
12821 }
12822
12823 #[tokio::test]
12827 async fn route_role_versions_capability_is_advertised() {
12828 let handler = ControlHandler::new(Arc::new(Registry::default()));
12829 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
12830 let ack = hello_via_sink(
12831 &handler,
12832 &ctx,
12833 &mut rx,
12834 hello_frame("m", PROTOCOL_VERSION, 1),
12835 )
12836 .await;
12837 let ack = parse_ack(&ack);
12838 assert!(
12839 ack.subc_capabilities
12840 .iter()
12841 .any(|c| c == "route-role-versions/v1"),
12842 "{:?}",
12843 ack.subc_capabilities
12844 );
12845
12846 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12847 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12848 let reply = handler
12849 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
12850 .await
12851 .unwrap()
12852 .pop()
12853 .unwrap();
12854 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12855 serde_json::from_slice(&reply.body).unwrap()
12856 else {
12857 panic!("not a server.describe reply");
12858 };
12859 assert!(
12860 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
12861 "{capabilities:?}"
12862 );
12863 }
12864
12865 #[tokio::test]
12869 async fn catalog_events_capability_is_advertised() {
12870 let handler = ControlHandler::new(Arc::new(Registry::default()));
12871 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12872 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12873 let reply = handler
12874 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
12875 .await
12876 .unwrap()
12877 .pop()
12878 .unwrap();
12879 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12880 serde_json::from_slice(&reply.body).unwrap()
12881 else {
12882 panic!("not a server.describe reply");
12883 };
12884 assert!(
12885 capabilities
12886 .iter()
12887 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
12888 "{capabilities:?}"
12889 );
12890 }
12891
12892 #[tokio::test]
12893 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
12894 let handler = ControlHandler::new(Arc::new(Registry::default()));
12895 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
12896 let ack = hello_via_sink(
12897 &handler,
12898 &module,
12899 &mut module_rx,
12900 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
12901 )
12902 .await;
12903 let ack = parse_ack(&ack);
12904 assert!(
12905 ack.subc_capabilities
12906 .iter()
12907 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12908 "{:?}",
12909 ack.subc_capabilities
12910 );
12911 assert!(ack
12912 .subc_capabilities
12913 .iter()
12914 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12915
12916 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
12917 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12918 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12919 let reply = handler
12920 .handle_control_frame(&client, frame)
12921 .await
12922 .unwrap()
12923 .pop()
12924 .unwrap();
12925 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12926 serde_json::from_slice(&reply.body).unwrap()
12927 else {
12928 panic!("not a server.describe reply");
12929 };
12930 assert!(
12931 capabilities
12932 .iter()
12933 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
12934 "{capabilities:?}"
12935 );
12936 assert!(capabilities
12937 .iter()
12938 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
12939 }
12940
12941 #[tokio::test]
12942 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
12943 let handler = ControlHandler::new(Arc::new(Registry::default()));
12944 let (module, mut rx) = route_ctx(ConnectionId::new(156));
12945 let ack = hello_via_sink(
12946 &handler,
12947 &module,
12948 &mut rx,
12949 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
12950 )
12951 .await;
12952 assert!(parse_ack(&ack)
12953 .subc_capabilities
12954 .iter()
12955 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12956 assert!(parse_ack(&ack)
12957 .subc_capabilities
12958 .iter()
12959 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12960 let (client, _rx) = route_ctx(ConnectionId::new(157));
12961 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
12962 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
12963 let reply = handler
12964 .handle_control_frame(&client, frame)
12965 .await
12966 .unwrap()
12967 .pop()
12968 .unwrap();
12969 let ClientControlResponse::ServerDescribe { capabilities, .. } =
12970 serde_json::from_slice(&reply.body).unwrap()
12971 else {
12972 panic!("server.describe reply")
12973 };
12974 assert!(capabilities
12975 .iter()
12976 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
12977 assert!(capabilities
12978 .iter()
12979 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
12980 }
12981
12982 #[tokio::test]
12983 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
12984 let handler = ControlHandler::new(Arc::new(Registry::default()));
12985 let (module, mut rx) = route_ctx(ConnectionId::new(158));
12986 let ack = hello_via_sink(
12987 &handler,
12988 &module,
12989 &mut rx,
12990 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
12991 )
12992 .await;
12993 assert!(parse_ack(&ack)
12994 .subc_capabilities
12995 .iter()
12996 .any(|cap| cap == "scope-workdir/v1"));
12997 let (client, _rx) = route_ctx(ConnectionId::new(159));
12998 let frame = Frame::build(
12999 FrameType::Request,
13000 control_flags(),
13001 0,
13002 0,
13003 5,
13004 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13005 )
13006 .unwrap();
13007 let reply = handler
13008 .handle_control_frame(&client, frame)
13009 .await
13010 .unwrap()
13011 .remove(0);
13012 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13013 serde_json::from_slice(&reply.body).unwrap()
13014 else {
13015 panic!("server.describe reply")
13016 };
13017 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
13018 }
13019
13020 #[tokio::test]
13021 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
13022 let handler = ControlHandler::new(Arc::new(Registry::default()));
13023 let (module, mut rx) = route_ctx(ConnectionId::new(160));
13024 let ack = hello_via_sink(
13025 &handler,
13026 &module,
13027 &mut rx,
13028 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
13029 )
13030 .await;
13031 assert!(parse_ack(&ack)
13032 .subc_capabilities
13033 .iter()
13034 .any(|cap| cap == "worker-runs/v1"));
13035 let (client, _rx) = route_ctx(ConnectionId::new(161));
13036 let frame = Frame::build(
13037 FrameType::Request,
13038 control_flags(),
13039 0,
13040 0,
13041 5,
13042 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13043 )
13044 .unwrap();
13045 let reply = handler
13046 .handle_control_frame(&client, frame)
13047 .await
13048 .unwrap()
13049 .remove(0);
13050 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13051 serde_json::from_slice(&reply.body).unwrap()
13052 else {
13053 panic!("server.describe reply")
13054 };
13055 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
13056 }
13057
13058 #[tokio::test]
13059 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
13060 let handler = ControlHandler::new(Arc::new(Registry::default()));
13061 let (module, mut rx) = route_ctx(ConnectionId::new(154));
13062 let ack = hello_via_sink(
13063 &handler,
13064 &module,
13065 &mut rx,
13066 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
13067 )
13068 .await;
13069 assert!(parse_ack(&ack)
13070 .subc_capabilities
13071 .iter()
13072 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13073 let (client, _rx) = route_ctx(ConnectionId::new(155));
13074 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13075 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13076 let reply = handler
13077 .handle_control_frame(&client, frame)
13078 .await
13079 .unwrap()
13080 .pop()
13081 .unwrap();
13082 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13083 serde_json::from_slice(&reply.body).unwrap()
13084 else {
13085 panic!("not a server.describe reply")
13086 };
13087 assert!(capabilities
13088 .iter()
13089 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13090 }
13091
13092 #[tokio::test]
13093 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
13094 let registry = Arc::new(Registry::default());
13095 let forwarding = Arc::new(ForwardingTable::default());
13096 let handler =
13097 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13098 .with_health_probe_timeout(Duration::from_secs(5));
13099 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
13100 hello_via_sink(
13101 &handler,
13102 &module_ctx,
13103 &mut module_rx,
13104 non_routable_hello_frame_with_control_ops(
13105 "mcp",
13106 300,
13107 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13108 ),
13109 )
13110 .await;
13111 assert!(registry
13112 .get_module("mcp")
13113 .unwrap()
13114 .unwrap()
13115 .manifest
13116 .provides
13117 .is_empty());
13118
13119 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
13120 let route_response = handler
13121 .handle_control_frame(
13122 &route_client_ctx,
13123 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
13124 )
13125 .await
13126 .unwrap();
13127 assert_eq!(route_response[0].header.ty, FrameType::Error);
13128 assert_eq!(
13129 parse_error(&route_response[0])["code"],
13130 "target_unavailable"
13131 );
13132 assert!(parse_error(&route_response[0])["message"]
13133 .as_str()
13134 .unwrap()
13135 .contains("does not provide the requested target"));
13136 assert!(module_rx.try_recv().is_err());
13137
13138 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
13139 let health_handler = handler.clone();
13140 let health_task = tokio::spawn(async move {
13141 health_handler
13142 .handle_control_frame(
13143 &health_client_ctx,
13144 supervisor_health_probe_frame(302, "mcp"),
13145 )
13146 .await
13147 .unwrap()
13148 });
13149 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13150 .await
13151 .unwrap()
13152 .unwrap();
13153 assert_eq!(
13154 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13155 ModuleControlRequest::HealthCheck {}
13156 );
13157 handler
13158 .handle_control_frame(
13159 &module_ctx,
13160 health_response(health_frame.header.corr, HealthStatus::Ok),
13161 )
13162 .await
13163 .unwrap();
13164 let health_response = health_task.await.unwrap();
13165 assert_eq!(health_response[0].header.ty, FrameType::Response);
13166 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13167 ClientControlResponse::SupervisorHealthProbe {
13168 module_id, status, ..
13169 } => {
13170 assert_eq!(module_id, "mcp");
13171 assert_eq!(status, HealthStatus::Ok);
13172 }
13173 other => panic!("unexpected health response: {other:?}"),
13174 }
13175
13176 forwarding
13181 .cleanup_connection(module_ctx.connection_id)
13182 .unwrap();
13183 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
13184 let cleanup_response = tokio::time::timeout(
13185 Duration::from_millis(200),
13186 handler.handle_control_frame(
13187 &cleanup_probe_ctx,
13188 supervisor_health_probe_frame(303, "mcp"),
13189 ),
13190 )
13191 .await
13192 .expect("probe should fail immediately when the control lane is gone")
13193 .unwrap();
13194 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
13195 assert_eq!(
13196 parse_error(&cleanup_response[0])["code"],
13197 "target_unavailable"
13198 );
13199 assert!(parse_error(&cleanup_response[0])["message"]
13200 .as_str()
13201 .unwrap()
13202 .contains("no module connection"));
13203
13204 handler
13205 .cleanup_connection(module_ctx.connection_id)
13206 .unwrap();
13207 }
13208
13209 #[tokio::test]
13210 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
13211 let registry = Arc::new(Registry::default());
13212 let supervisor_handle = SupervisorHandle::new();
13213 let supervisor =
13214 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13215 .with_handle(supervisor_handle.clone())
13216 .with_connection_file_path(
13217 std::env::temp_dir()
13218 .join(format!("subc-route-open-warming-{}", std::process::id())),
13219 );
13220 let module = supervisor
13221 .supervise_configured(
13222 ModuleSpec {
13223 module_id: "warming".to_string(),
13224 program: fake_aft_stub_path(),
13225 args: Vec::new(),
13226 env: Vec::new(),
13227 reserved: false,
13228 reserved_prefixes: Vec::new(),
13229 protocol: ModuleProtocol::Subc,
13230 overlap: Default::default(),
13231 },
13232 true,
13233 )
13234 .unwrap();
13235 assert_eq!(module.state().unwrap(), ModuleState::Running);
13236
13237 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13238 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13239 let response = handler
13240 .handle_control_frame(
13241 &ctx,
13242 route_open_frame(304, "warming", unique_project_root("warming")),
13243 )
13244 .await
13245 .unwrap();
13246 module.stop().await.unwrap();
13247
13248 assert_eq!(response[0].header.ty, FrameType::Error);
13249 let error = parse_error(&response[0]);
13250 assert_eq!(error["code"], "module_warming");
13251 assert!(error["message"]
13252 .as_str()
13253 .unwrap()
13254 .contains("state=running, enabled=true, live=false"));
13255 }
13256
13257 #[test]
13258 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13259 let handler = ControlHandler::new(Arc::new(Registry::default()));
13260 let capture = EventCapture::default();
13261 let _subscriber =
13262 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13263 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13264 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
13265 let pending = (0..limit).collect::<Vec<_>>();
13266 let response = handler
13267 .route_open_capacity_refusal(
13268 &ctx,
13269 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
13270 "busy",
13271 pending.len(),
13272 limit,
13273 )
13274 .unwrap();
13275 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13276 let event = capture
13277 .events()
13278 .into_iter()
13279 .find(|event| {
13280 event.target == "control"
13281 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
13282 })
13283 .expect("connection admission refusal event");
13284 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13285 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13286 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13289 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13290 }
13291
13292 #[test]
13293 fn route_open_target_cap_logs_admission_reason_and_capacity() {
13294 let handler = ControlHandler::new(Arc::new(Registry::default()));
13295 let capture = EventCapture::default();
13296 let _subscriber =
13297 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13298 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
13299 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
13300 let guards = (0..limit)
13301 .map(|_| {
13302 handler
13303 .route_bind_concurrency
13304 .try_admit("busy", limit)
13305 .unwrap()
13306 })
13307 .collect::<Vec<_>>();
13308 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
13309 Err(in_flight) => in_flight,
13310 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
13311 };
13312 let response = handler
13313 .route_open_target_capacity_refusal(
13314 &ctx,
13315 &RouteOpenCaller::unknown(),
13316 &route_open_frame(397, "busy", unique_project_root("target-cap")),
13317 "busy",
13318 in_flight,
13319 )
13320 .unwrap();
13321 assert_eq!(parse_error(&response)["code"], "target_unavailable");
13322 let event = capture
13323 .events()
13324 .into_iter()
13325 .find(|event| {
13326 event.target == "control"
13327 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
13328 })
13329 .expect("target admission refusal event");
13330 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
13331 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
13332 drop(guards);
13333 }
13334
13335 #[cfg(unix)]
13336 #[tokio::test(start_paused = true)]
13337 async fn supervisor_list_shows_a_failed_modules_next_retry() {
13338 let registry = Arc::new(Registry::default());
13339 let handle = SupervisorHandle::new();
13340 let supervisor =
13341 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
13342 .with_handle(handle.clone());
13343 let module = supervisor
13344 .spawn(crate::ModuleSpec {
13345 module_id: "retry-list".into(),
13346 program: PathBuf::from("/bin/sh"),
13347 args: vec!["-c".into(), "exit 17".into()],
13348 env: vec![],
13349 reserved: false,
13350 reserved_prefixes: vec![],
13351 protocol: ModuleProtocol::None,
13352 overlap: Default::default(),
13353 })
13354 .unwrap();
13355 let deadline = std::time::Instant::now() + Duration::from_secs(10);
13356 while module.state().unwrap() != crate::ModuleState::Failed {
13357 assert!(
13358 std::time::Instant::now() < deadline,
13359 "waiting for budget failure"
13360 );
13361 tokio::task::yield_now().await;
13362 }
13363 let handler = ControlHandler::new(registry).with_supervisor(handle);
13364 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13365 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13366 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13367 assert_eq!(body["modules"][0]["state"], "failed");
13368 let retry = body["modules"][0]["next_retry_at_ms"]
13369 .as_u64()
13370 .expect("scheduled retry on the wire");
13371 let now = std::time::SystemTime::now()
13372 .duration_since(std::time::UNIX_EPOCH)
13373 .unwrap()
13374 .as_millis() as u64;
13375 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
13376 module.set_enabled(false).await.unwrap();
13377 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13378 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13379 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13380 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
13381 }
13382
13383 #[tokio::test]
13384 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
13385 let registry = Arc::new(Registry::default());
13386 let handle = SupervisorHandle::new();
13387 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
13388 .with_handle(handle.clone());
13389 supervisor
13390 .supervise_configured(
13391 crate::ModuleSpec {
13392 module_id: "drift".into(),
13393 program: PathBuf::from("disabled-module"),
13394 args: Vec::new(),
13395 env: Vec::new(),
13396 reserved: false,
13397 reserved_prefixes: Vec::new(),
13398 protocol: ModuleProtocol::Subc,
13399 overlap: Default::default(),
13400 },
13401 false,
13402 )
13403 .unwrap();
13404 let handler = ControlHandler::new(registry).with_supervisor(handle);
13405 for configured in [None, Some(false), Some(true)] {
13406 handler.capability_evaluator.configure(
13407 configured.map(|enabled| ("drift".to_string(), enabled)),
13408 BTreeMap::new(),
13409 );
13410 let frame =
13411 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13412 let replies = handler
13413 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
13414 .await
13415 .unwrap();
13416 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13417 assert!(body.get("configured_enabled").is_none());
13418 assert_eq!(body["op"], "supervisor.ack");
13419 let frame =
13420 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13421 let replies = handler.handle_supervisor_list(frame).await.unwrap();
13422 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
13423 assert_eq!(
13424 body["modules"][0].get("configured_enabled"),
13425 configured.map(Value::Bool).as_ref()
13426 );
13427 }
13428 }
13429
13430 #[tokio::test(flavor = "current_thread")]
13431 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
13432 let handler = ControlHandler::new(Arc::new(Registry::default()));
13433 let capture = EventCapture::default();
13434 let _subscriber =
13435 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13436 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
13437 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13438
13439 let _ = handler
13440 .handle_client_control_request(
13441 &ctx,
13442 frame,
13443 ClientControlRequest::SupervisorSetEnabled {
13444 module_id: "broca".to_string(),
13445 enabled: false,
13446 },
13447 &mut None,
13448 )
13449 .await
13450 .unwrap();
13451
13452 let events = capture
13453 .events()
13454 .into_iter()
13455 .filter(|event| {
13456 event.target == "control"
13457 && event.fields.get("message").map(String::as_str)
13458 == Some("supervisor request received")
13459 })
13460 .collect::<Vec<_>>();
13461 assert_eq!(events.len(), 1, "one supervisor request log line");
13462 let event = &events[0];
13463 assert_eq!(event.level, tracing::Level::INFO);
13464 assert_eq!(
13465 event.fields.get("op"),
13466 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
13467 );
13468 assert_eq!(
13469 event.fields.get("module_id"),
13470 Some(&"\"broca\"".to_string())
13471 );
13472 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
13473 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
13474 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
13475 }
13476
13477 #[tokio::test(flavor = "current_thread")]
13478 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
13479 let registry = Arc::new(Registry::default());
13480 let module_connection = ConnectionId::new(303);
13481 registry
13482 .register_with_control_ops(
13483 manifest("aft", PROTOCOL_VERSION),
13484 PROTOCOL_VERSION,
13485 module_connection,
13486 module_baseline_control_ops(),
13487 )
13488 .unwrap();
13489 let handler = ControlHandler::new(registry);
13490 let capture = EventCapture::default();
13491 let _subscriber =
13492 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13493 let (ctx, _rx) = route_ctx(module_connection);
13494 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
13495
13496 let _ = handler
13497 .handle_client_control_request(
13498 &ctx,
13499 frame,
13500 ClientControlRequest::SupervisorSetEnabled {
13501 module_id: "broca".to_string(),
13502 enabled: false,
13503 },
13504 &mut None,
13505 )
13506 .await
13507 .unwrap();
13508
13509 let callers = capture
13510 .events()
13511 .into_iter()
13512 .filter(|event| {
13513 event.fields.get("message").map(String::as_str)
13514 == Some("supervisor request received")
13515 })
13516 .map(|event| event.fields.get("caller").cloned())
13517 .collect::<Vec<_>>();
13518 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
13519 }
13520
13521 #[tokio::test(flavor = "current_thread")]
13522 async fn supervisor_list_does_not_log_request_received() {
13523 let handler = ControlHandler::new(Arc::new(Registry::default()));
13524 let capture = EventCapture::default();
13525 let _subscriber =
13526 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13527 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
13528 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
13529
13530 handler
13531 .handle_client_control_request(
13532 &ctx,
13533 frame,
13534 ClientControlRequest::SupervisorList {},
13535 &mut None,
13536 )
13537 .await
13538 .unwrap();
13539
13540 assert!(capture.events().into_iter().all(|event| {
13541 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13542 }));
13543 }
13544
13545 #[tokio::test(flavor = "current_thread")]
13546 async fn supervisor_rescan_preview_does_not_log_request_received() {
13547 let handler = ControlHandler::new(Arc::new(Registry::default()));
13548 let capture = EventCapture::default();
13549 let _subscriber =
13550 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13551 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
13552 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
13553
13554 handler
13555 .handle_client_control_request(
13556 &ctx,
13557 frame,
13558 ClientControlRequest::SupervisorRescan { preview: true },
13559 &mut None,
13560 )
13561 .await
13562 .unwrap();
13563
13564 assert!(capture.events().into_iter().all(|event| {
13565 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
13566 }));
13567 }
13568
13569 struct ApproveOperator;
13570 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
13571 fn prompt(
13572 &self,
13573 _: &str,
13574 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
13575 ) -> crate::operator_confirm::ProviderResult {
13576 crate::operator_confirm::ProviderResult::Approved
13577 }
13578 }
13579
13580 #[tokio::test]
13581 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
13582 assert_eq!(
13583 ControlHandler::default().privileged_operator_prompt,
13584 PrivilegedOperatorPrompt::Log
13585 );
13586 for (initial, changed, running) in [
13587 ("log", "enforce", PrivilegedOperatorPrompt::Log),
13588 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
13589 ] {
13590 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
13591 let path = dir.join("subc.jsonc");
13592 std::fs::write(
13593 &path,
13594 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
13595 )
13596 .unwrap();
13597 let config = crate::daemon_config::load(&path).unwrap().unwrap();
13598 let registry = Arc::new(Registry::default());
13599 let handle = SupervisorHandle::new();
13600 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
13601 .with_handle(handle.clone());
13602 let handler = ControlHandler::new(registry)
13603 .with_supervisor(handle)
13604 .with_privileged_operator_prompt(config.privileged_operator_prompt)
13605 .with_supervisor_rescan(supervisor, &path, config.port);
13606 handler.forwarding.operator_confirms().configure(
13607 Arc::new(ApproveOperator),
13608 crate::operator_confirm::OperatorLimits::default(),
13609 );
13610 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
13611
13612 for (disk_mode, needs_restart) in [
13613 (Some(initial), false),
13614 (Some(changed), true),
13615 (Some(changed), true),
13616 (None, running == PrivilegedOperatorPrompt::Enforce),
13617 (Some(initial), false),
13618 ] {
13619 let mut doc = json!({"version":1});
13620 if let Some(mode) = disk_mode {
13621 doc["privileged_operator_prompt"] = json!(mode);
13622 }
13623 std::fs::write(&path, doc.to_string()).unwrap();
13624 for preview in [true, false] {
13625 let request = ClientControlRequest::SupervisorRescan { preview };
13626 let frame = Frame::build(
13627 FrameType::Request,
13628 control_flags(),
13629 0,
13630 0,
13631 1,
13632 serde_json::to_vec(&request).unwrap(),
13633 )
13634 .unwrap();
13635 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13636 assert_eq!(frames.len(), 1);
13637 let ClientControlResponse::SupervisorRescan { result } =
13638 serde_json::from_slice(&frames[0].body).unwrap()
13639 else {
13640 panic!("expected a rescan response");
13641 };
13642 let expected = if needs_restart {
13643 vec!["privileged_operator_prompt"]
13644 } else {
13645 Vec::new()
13646 };
13647 assert_eq!(result.restart_required, expected);
13648 assert_eq!(handler.privileged_operator_prompt, running);
13649 assert_eq!(
13650 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
13651 running
13652 );
13653 }
13654 }
13655 }
13656 }
13657
13658 #[tokio::test]
13659 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
13660 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
13661 let path = dir.join("subc.jsonc");
13662 let registry = Arc::new(Registry::default());
13663 let handle = SupervisorHandle::new();
13664 let supervisor =
13665 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
13666 let handler = ControlHandler::new(registry)
13667 .with_supervisor(handle)
13668 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
13669 .with_supervisor_rescan(supervisor, &path, None);
13670 handler.forwarding.operator_confirms().configure(
13671 Arc::new(ApproveOperator),
13672 crate::operator_confirm::OperatorLimits::default(),
13673 );
13674 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
13675 for preview in [true, false] {
13676 std::fs::write(
13677 &path,
13678 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
13679 )
13680 .unwrap();
13681 let request = ClientControlRequest::SupervisorRescan { preview };
13682 let frame = Frame::build(
13683 FrameType::Request,
13684 control_flags(),
13685 0,
13686 0,
13687 1,
13688 serde_json::to_vec(&request).unwrap(),
13689 )
13690 .unwrap();
13691 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
13692 assert_eq!(frames.len(), 1);
13693 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
13694 assert_eq!(
13695 handler.privileged_operator_prompt,
13696 PrivilegedOperatorPrompt::Enforce
13697 );
13698 }
13699 }
13700
13701 #[tokio::test]
13706 async fn route_open_refusal_names_the_check_that_refused() {
13707 let handler = ControlHandler::new(Arc::new(Registry::default()));
13708 let capture = EventCapture::default();
13709 let _subscriber =
13710 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13711 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
13712 let response = handler
13713 .handle_control_frame(
13714 &ctx,
13715 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
13716 )
13717 .await
13718 .unwrap();
13719
13720 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13721 let event = capture
13722 .events()
13723 .into_iter()
13724 .find(|event| {
13725 event.target == "control"
13726 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
13727 })
13728 .expect("route.open refusal event");
13729 assert_eq!(
13730 event.fields.get("reason"),
13731 Some(&"\"not_registered\"".to_string())
13732 );
13733 }
13734
13735 #[tokio::test]
13736 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
13737 let registry = Arc::new(Registry::default());
13738 let supervisor_handle = SupervisorHandle::new();
13739 let supervisor =
13740 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13741 .with_handle(supervisor_handle.clone())
13742 .with_connection_file_path(std::env::temp_dir().join(format!(
13743 "subc-route-open-refusal-info-{}",
13744 std::process::id()
13745 )));
13746 let module = supervisor
13747 .supervise_configured(
13748 ModuleSpec {
13749 module_id: "warming".to_string(),
13750 program: fake_aft_stub_path(),
13751 args: Vec::new(),
13752 env: Vec::new(),
13753 reserved: false,
13754 reserved_prefixes: Vec::new(),
13755 protocol: ModuleProtocol::Subc,
13756 overlap: Default::default(),
13757 },
13758 true,
13759 )
13760 .unwrap();
13761 assert_eq!(module.state().unwrap(), ModuleState::Running);
13762
13763 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13764 assert!(handler
13765 .counters()
13766 .snapshot()
13767 .get("route_open_refused_by_code")
13768 .is_none());
13769 let capture = EventCapture::default();
13770 let _subscriber =
13771 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13772 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
13773 let response = handler
13774 .handle_control_frame(
13775 &ctx,
13776 route_open_frame(394, "warming", unique_project_root("refusal-info")),
13777 )
13778 .await
13779 .unwrap();
13780 module.stop().await.unwrap();
13781
13782 assert_eq!(parse_error(&response[0])["code"], "module_warming");
13783 let event = capture
13784 .events()
13785 .into_iter()
13786 .find(|event| {
13787 event.target == "control"
13788 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
13789 })
13790 .expect("route.open refusal event");
13791 assert_eq!(
13792 event.fields.get("module_id"),
13793 Some(&"\"warming\"".to_string())
13794 );
13795 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
13796 assert_eq!(
13797 event.fields.get("reason"),
13798 Some(&"\"supervised_not_registered\"".to_string())
13799 );
13800 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
13801 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
13802 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
13803 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
13804 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
13805 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
13806 assert_eq!(
13807 handler.counters().snapshot()["route_open_refused_by_code"],
13808 json!({ "module_warming": 1 })
13809 );
13810 }
13811
13812 const OUTAGE_START: &str = "route.open refusing module: not serving";
13813 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
13814
13815 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
13816 capture
13817 .events()
13818 .into_iter()
13819 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
13820 .collect()
13821 }
13822
13823 fn supervise_stub(
13824 registry: &Arc<Registry>,
13825 module_id: &str,
13826 enabled: bool,
13827 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
13828 let supervisor_handle = SupervisorHandle::new();
13829 let supervisor =
13830 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
13831 .with_handle(supervisor_handle.clone())
13832 .with_connection_file_path(std::env::temp_dir().join(format!(
13833 "subc-route-outage-{module_id}-{}",
13834 std::process::id()
13835 )));
13836 let module = supervisor
13837 .supervise_configured(
13838 ModuleSpec {
13839 module_id: module_id.to_string(),
13840 program: fake_aft_stub_path(),
13841 args: Vec::new(),
13842 env: Vec::new(),
13843 reserved: false,
13844 reserved_prefixes: Vec::new(),
13845 protocol: ModuleProtocol::Subc,
13846 overlap: Default::default(),
13847 },
13848 enabled,
13849 )
13850 .unwrap();
13851 (supervisor_handle, module)
13852 }
13853
13854 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
13855 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
13856 module_id: module_id.to_string(),
13857 drain_timeout_ms: Some(50),
13858 })
13859 .unwrap();
13860 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
13861 }
13862
13863 #[test]
13867 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
13868 let registry = Arc::new(Registry::default());
13869 let forwarding = Arc::new(ForwardingTable::default());
13870 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13871 let second = ControlHandler::with_forwarding(registry, forwarding);
13872 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
13873 }
13874
13875 #[tokio::test(flavor = "current_thread")]
13879 async fn route_open_unknown_module_refusals_add_no_outage_state() {
13880 let handler = ControlHandler::new(Arc::new(Registry::default()));
13881 let capture = EventCapture::default();
13882 let _subscriber =
13883 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13884 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
13885 for corr in 0..8 {
13886 let response = handler
13887 .handle_control_frame(
13888 &ctx,
13889 route_open_frame(
13890 380 + corr,
13891 &format!("nobody-{corr}"),
13892 unique_project_root("outage-unknown"),
13893 ),
13894 )
13895 .await
13896 .unwrap();
13897 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
13898 }
13899
13900 assert_eq!(handler.route_outages.tracked_module_count(), 0);
13901 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
13902 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
13903 }
13904
13905 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13910 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
13911 let registry = Arc::new(Registry::default());
13912 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
13913 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13914 let capture = EventCapture::default();
13915 let _subscriber =
13916 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13917 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
13918 handler.route_outages.record_accepted("outage-restart");
13921
13922 let response = handler
13923 .handle_control_frame(
13924 &ctx,
13925 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
13926 )
13927 .await
13928 .unwrap();
13929 assert_eq!(response[0].header.ty, FrameType::Error);
13930 let starts = outage_lines(&capture, OUTAGE_START);
13931 assert_eq!(starts.len(), 1, "{starts:?}");
13932 assert_eq!(starts[0].level, tracing::Level::WARN);
13933 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
13934 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
13935 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
13936 handler.route_outages.record_accepted("outage-restart");
13937 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
13938
13939 let restart = handler
13940 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
13941 .await
13942 .unwrap();
13943 assert_eq!(
13944 restart[0].header.ty,
13945 FrameType::Response,
13946 "{:?}",
13947 parse_error(&restart[0])
13948 );
13949 handler
13950 .handle_control_frame(
13951 &ctx,
13952 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
13953 )
13954 .await
13955 .unwrap();
13956 module.stop().await.unwrap();
13957
13958 let starts = outage_lines(&capture, OUTAGE_START);
13959 assert_eq!(starts.len(), 2, "{starts:?}");
13960 assert_eq!(starts[1].level, tracing::Level::INFO);
13961 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
13962 }
13963
13964 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
13968 async fn failed_operator_restart_leaves_no_operator_mark() {
13969 let registry = Arc::new(Registry::default());
13970 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
13971 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13972 let capture = EventCapture::default();
13973 let _subscriber =
13974 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13975 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
13976 handler.route_outages.record_accepted("outage-disabled");
13977
13978 let restart = handler
13979 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
13980 .await
13981 .unwrap();
13982 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
13983 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
13984
13985 handler
13986 .handle_control_frame(
13987 &ctx,
13988 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
13989 )
13990 .await
13991 .unwrap();
13992 let starts = outage_lines(&capture, OUTAGE_START);
13993 assert_eq!(starts.len(), 1, "{starts:?}");
13994 assert_eq!(starts[0].level, tracing::Level::WARN);
13995 }
13996
13997 #[tokio::test(flavor = "current_thread")]
13998 async fn route_open_unknown_module_escapes_target_module_id() {
13999 let handler = ControlHandler::new(Arc::new(Registry::default()));
14000 let capture = EventCapture::default();
14001 let _subscriber =
14002 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14003 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
14004 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14005 let response = handler
14006 .handle_control_frame(
14007 &ctx,
14008 route_open_frame(
14009 395,
14010 hostile_module_id,
14011 unique_project_root("hostile-target-module-id"),
14012 ),
14013 )
14014 .await
14015 .unwrap();
14016
14017 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14018 let event = capture
14019 .events()
14020 .into_iter()
14021 .find(|event| {
14022 event.target == "control"
14023 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14024 })
14025 .expect("route.open unknown-module refusal event");
14026 let logged = event.fields.get("module_id").expect("module_id field");
14027 assert!(!logged.bytes().any(|byte| byte < 0x20));
14028 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14029 }
14030
14031 #[tokio::test(flavor = "current_thread")]
14034 async fn route_open_refusal_carries_daemon_origin() {
14035 let handler = ControlHandler::new(Arc::new(Registry::default()));
14036 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14037 let response = handler
14038 .handle_control_frame(
14039 &ctx,
14040 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
14041 )
14042 .await
14043 .unwrap();
14044
14045 assert_eq!(response.len(), 1);
14046 assert_eq!(response[0].header.ty, FrameType::Error);
14047 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14048 assert!(
14049 response[0].header.flags.is_daemon_origin(),
14050 "route.open refusal must carry daemon origin: {:?}",
14051 response[0].header.flags
14052 );
14053 }
14054
14055 #[tokio::test(flavor = "current_thread")]
14059 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
14060 let registry = Arc::new(Registry::default());
14061 let forwarding = Arc::new(ForwardingTable::default());
14062 let handler =
14063 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14064 let module_connection = ConnectionId::new(95);
14065 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14066 hello_via_sink(
14067 &handler,
14068 &module_ctx,
14069 &mut module_rx,
14070 hello_frame("aft", PROTOCOL_VERSION, 395),
14071 )
14072 .await;
14073 forwarding
14074 .begin_module_drain("aft", RouteCloseReason::Restart)
14075 .unwrap()
14076 .expect("the module endpoint drains");
14077
14078 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
14079 let response = handler
14080 .handle_control_frame(
14081 &client_ctx,
14082 route_open_frame(396, "aft", unique_project_root("drain-origin")),
14083 )
14084 .await
14085 .unwrap();
14086
14087 assert_eq!(response.len(), 1);
14088 assert_eq!(response[0].header.ty, FrameType::Error);
14089 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14090 assert!(
14091 response[0].header.flags.is_daemon_origin(),
14092 "drain module_reloading must carry daemon origin: {:?}",
14093 response[0].header.flags
14094 );
14095 }
14096
14097 #[tokio::test(flavor = "current_thread")]
14101 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
14102 let registry = Arc::new(Registry::default());
14103 let forwarding = Arc::new(ForwardingTable::default());
14104 let handler =
14105 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14106 let module_connection = ConnectionId::new(95);
14107 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14108 hello_via_sink(
14109 &handler,
14110 &module_ctx,
14111 &mut module_rx,
14112 hello_frame("aft", PROTOCOL_VERSION, 395),
14113 )
14114 .await;
14115
14116 let client_connection = ConnectionId::new(96);
14117 let (client_ctx, _client_rx) = route_ctx(client_connection);
14118 let (route_task, bind) = relay_route_open(
14119 &handler,
14120 client_connection,
14121 &client_ctx.egress,
14122 &mut module_rx,
14123 396,
14124 "aft",
14125 "spoofed-reloading",
14126 )
14127 .await;
14128 let rejection = Frame::build(
14131 FrameType::Error,
14132 crate::router::daemon_error_flags(),
14133 0,
14134 0,
14135 bind.header.corr,
14136 serde_json::to_vec(&ErrorBody::new(
14137 "module_reloading",
14138 "module refused route.bind",
14139 ))
14140 .unwrap(),
14141 )
14142 .unwrap();
14143 handler
14144 .handle_control_frame(&module_ctx, rejection)
14145 .await
14146 .unwrap();
14147
14148 let response = route_task.await.unwrap();
14149 assert_eq!(response.len(), 1);
14150 assert_eq!(response[0].header.ty, FrameType::Error);
14151 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
14152 assert!(
14153 !response[0].header.flags.is_daemon_origin(),
14154 "a module's own rejection must not carry daemon origin: {:?}",
14155 response[0].header.flags
14156 );
14157 }
14158
14159 #[tokio::test(flavor = "current_thread")]
14160 async fn route_open_module_rejection_uses_daemon_counter_key() {
14161 let registry = Arc::new(Registry::default());
14162 let forwarding = Arc::new(ForwardingTable::default());
14163 let handler =
14164 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14165 let module_connection = ConnectionId::new(95);
14166 let (module_ctx, mut module_rx) = route_ctx(module_connection);
14167 hello_via_sink(
14168 &handler,
14169 &module_ctx,
14170 &mut module_rx,
14171 hello_frame("aft", PROTOCOL_VERSION, 395),
14172 )
14173 .await;
14174
14175 let client_connection = ConnectionId::new(96);
14176 let (client_ctx, _client_rx) = route_ctx(client_connection);
14177 let capture = EventCapture::default();
14178 let _subscriber =
14179 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14180 let (route_task, bind) = relay_route_open(
14181 &handler,
14182 client_connection,
14183 &client_ctx.egress,
14184 &mut module_rx,
14185 396,
14186 "aft",
14187 "hostile-module-code",
14188 )
14189 .await;
14190 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
14191 let rejection = Frame::build(
14192 FrameType::Error,
14193 control_flags(),
14194 0,
14195 0,
14196 bind.header.corr,
14197 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
14198 )
14199 .unwrap();
14200 handler
14201 .handle_control_frame(&module_ctx, rejection)
14202 .await
14203 .unwrap();
14204
14205 let response = route_task.await.unwrap();
14206 assert_eq!(parse_error(&response[0])["code"], hostile_code);
14207 let counters = handler.counters().snapshot();
14208 assert_eq!(
14209 counters["route_open_refused_by_code"],
14210 json!({ "module_rejected": 1 })
14211 );
14212 assert!(counters["route_open_refused_by_code"]
14213 .get(hostile_code)
14214 .is_none());
14215
14216 let event = capture
14217 .events()
14218 .into_iter()
14219 .find(|event| {
14220 event.target == "control"
14221 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
14222 })
14223 .expect("route.open module-rejection refusal event");
14224 let logged = event.fields.get("module_code").expect("module_code field");
14225 assert!(!logged.bytes().any(|byte| byte < 0x20));
14226 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
14227 }
14228
14229 #[tokio::test]
14230 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
14231 let registry = Arc::new(Registry::default());
14232 let supervisor_handle = SupervisorHandle::new();
14233 let missing_program = std::env::temp_dir().join(format!(
14234 "subc-route-open-missing-program-{}",
14235 std::process::id()
14236 ));
14237 let supervisor =
14238 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
14239 .with_handle(supervisor_handle.clone());
14240 let module = supervisor
14241 .supervise_configured(
14242 ModuleSpec {
14243 module_id: "failed".to_string(),
14244 program: missing_program,
14245 args: Vec::new(),
14246 env: Vec::new(),
14247 reserved: false,
14248 reserved_prefixes: Vec::new(),
14249 protocol: ModuleProtocol::Subc,
14250 overlap: Default::default(),
14251 },
14252 true,
14253 )
14254 .unwrap();
14255 assert_eq!(module.state().unwrap(), ModuleState::Failed);
14256
14257 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14258 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
14259 let response = handler
14260 .handle_control_frame(
14261 &ctx,
14262 route_open_frame(305, "failed", unique_project_root("failed")),
14263 )
14264 .await
14265 .unwrap();
14266
14267 assert_eq!(response[0].header.ty, FrameType::Error);
14268 let error = parse_error(&response[0]);
14269 assert_eq!(error["code"], "target_unavailable");
14270 assert!(error["message"]
14271 .as_str()
14272 .unwrap()
14273 .contains("state=failed, enabled=true, live=false"));
14274 }
14275
14276 #[tokio::test]
14277 async fn route_open_role_mismatch_remains_target_unavailable() {
14278 let registry = Arc::new(Registry::default());
14279 let handler = ControlHandler::new(Arc::clone(®istry));
14280 handler
14281 .handle_control(
14282 ConnectionId::new(41),
14283 non_routable_hello_frame_with_control_ops("health-only", 306, None),
14284 )
14285 .unwrap();
14286
14287 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
14288 let response = handler
14289 .handle_control_frame(
14290 &ctx,
14291 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
14292 )
14293 .await
14294 .unwrap();
14295
14296 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14297 assert!(parse_error(&response[0])["message"]
14298 .as_str()
14299 .unwrap()
14300 .contains("does not provide the requested target"));
14301 }
14302
14303 #[tokio::test]
14304 async fn route_open_inactive_registration_remains_target_unavailable() {
14305 let registry = Arc::new(Registry::default());
14306 let handler = ControlHandler::new(Arc::clone(®istry));
14307 handler
14308 .handle_control(
14309 ConnectionId::new(43),
14310 hello_frame("inactive", PROTOCOL_VERSION, 308),
14311 )
14312 .unwrap();
14313 assert!(registry
14314 .set_module_state_for_test("inactive", ChannelState::Closed)
14315 .unwrap());
14316
14317 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
14318 let response = handler
14319 .handle_control_frame(
14320 &ctx,
14321 route_open_frame(309, "inactive", unique_project_root("inactive")),
14322 )
14323 .await
14324 .unwrap();
14325
14326 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
14327 assert!(parse_error(&response[0])["message"]
14328 .as_str()
14329 .unwrap()
14330 .contains("is not active"));
14331 }
14332
14333 #[tokio::test]
14334 async fn late_health_reply_is_recorded_through_the_module_response_path() {
14335 let registry = Arc::new(Registry::default());
14336 let forwarding = Arc::new(ForwardingTable::default());
14337 let supervisor_handle = SupervisorHandle::new();
14338 let supervisor =
14339 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
14340 .with_forwarding(Arc::clone(&forwarding))
14341 .with_handle(supervisor_handle.clone());
14342 let module = supervisor
14343 .supervise_configured(
14344 crate::ModuleSpec {
14345 module_id: "late-health-response".to_string(),
14346 program: PathBuf::from("disabled-module"),
14347 args: Vec::new(),
14348 env: Vec::new(),
14349 reserved: false,
14350 reserved_prefixes: Vec::new(),
14351 protocol: ModuleProtocol::Subc,
14352 overlap: Default::default(),
14353 },
14354 false,
14355 )
14356 .unwrap();
14357 let handler =
14358 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14359 .with_supervisor(supervisor_handle);
14360 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
14361 handler
14362 .handle_control_frame(
14363 &module_ctx,
14364 hello_frame_with_control_ops(
14365 "late-health-response",
14366 PROTOCOL_VERSION,
14367 7,
14368 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14369 ),
14370 )
14371 .await
14372 .unwrap();
14373 let probe_started_at = Instant::now() - Duration::from_millis(80);
14374 let pending = forwarding
14375 .begin_health_probe_rpc_for(
14376 "late-health-response",
14377 MODULE_CONTROL_OP_HEALTH_CHECK,
14378 probe_started_at,
14379 Instant::now() - Duration::from_millis(1),
14380 )
14381 .unwrap();
14382 assert!(forwarding
14383 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
14384 .unwrap());
14385
14386 let responses = handler
14387 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
14388 .await
14389 .unwrap();
14390
14391 assert!(responses.is_empty());
14392 let health = module.status().unwrap().health;
14393 assert_eq!(health.late_answer_count, 1);
14394 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
14395 }
14396
14397 #[tokio::test]
14398 async fn health_probe_timeout_and_module_death_are_typed() {
14399 let registry = Arc::new(Registry::default());
14400 let forwarding = Arc::new(ForwardingTable::default());
14401 let handler =
14402 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14403 .with_health_probe_timeout(Duration::from_millis(50));
14404 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
14405 hello_via_sink(
14406 &handler,
14407 &module_ctx,
14408 &mut module_rx,
14409 hello_frame_with_control_ops(
14410 "aft",
14411 PROTOCOL_VERSION,
14412 7,
14413 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
14414 ),
14415 )
14416 .await;
14417
14418 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
14419 let responses = handler
14420 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
14421 .await
14422 .unwrap();
14423 assert_eq!(responses[0].header.ty, FrameType::Error);
14424 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
14425 let _ = module_rx.try_recv();
14426
14427 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
14428 let health_handler = handler.clone();
14429 let death_task = tokio::spawn(async move {
14430 health_handler
14431 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
14432 .await
14433 .unwrap()
14434 });
14435 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
14436 .await
14437 .unwrap()
14438 .unwrap();
14439 handler
14440 .cleanup_connection(module_ctx.connection_id)
14441 .unwrap();
14442 let responses = death_task.await.unwrap();
14443 assert_eq!(responses[0].header.ty, FrameType::Error);
14444 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
14445 }
14446
14447 #[test]
14448 fn hello_requires_exact_protocol_version() {
14449 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
14450 let registry = Arc::new(Registry::default());
14451 let handler = ControlHandler::new(Arc::clone(®istry));
14452 let responses = handler
14453 .handle_control(
14454 ConnectionId::new(connection),
14455 hello_frame("aft", offered, 9),
14456 )
14457 .unwrap();
14458
14459 assert_eq!(responses.len(), 1);
14460 assert_eq!(responses[0].header.ty, FrameType::Error);
14461 let error = parse_error(&responses[0]);
14462 assert_eq!(error["code"], "version_unsupported");
14463 assert!(registry.get_module("aft").unwrap().is_none());
14464 assert_eq!(registry.active_registration_count().unwrap(), 0);
14465 }
14466 }
14467
14468 #[test]
14469 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
14470 let registry = Arc::new(Registry::default());
14471 let forwarding = Arc::new(ForwardingTable::default());
14472 let handler =
14473 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14474 let module_connection = ConnectionId::new(301);
14475 let registration = registry
14476 .register_with_control_ops(
14477 manifest("aft-push", PROTOCOL_VERSION),
14478 PROTOCOL_VERSION,
14479 module_connection,
14480 module_baseline_control_ops(),
14481 )
14482 .unwrap();
14483 let (module_tx, _module_rx) = mpsc::channel(8);
14484 let endpoint = forwarding
14485 .register_module_connection(
14486 module_connection,
14487 "aft-push".to_string(),
14488 PROTOCOL_VERSION,
14489 manifest_concurrency(®istration.manifest),
14490 FrameSink::new(module_tx),
14491 )
14492 .unwrap();
14493
14494 let unknown = Frame::build(
14496 FrameType::Push,
14497 control_flags(),
14498 0,
14499 0,
14500 5,
14501 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
14502 )
14503 .unwrap();
14504 let out = handler.handle_status_update(endpoint, unknown).unwrap();
14505 assert!(
14506 out.is_empty(),
14507 "unknown push op must be ignored, got {out:?}"
14508 );
14509
14510 let malformed = Frame::build(
14512 FrameType::Push,
14513 control_flags(),
14514 0,
14515 0,
14516 6,
14517 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
14518 )
14519 .unwrap();
14520 let out = handler.handle_status_update(endpoint, malformed).unwrap();
14521 assert_eq!(out.len(), 1);
14522 assert_eq!(out[0].header.ty, FrameType::Error);
14523 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
14524 }
14525
14526 #[test]
14527 fn hello_rejected_when_connection_already_owns_client_routes() {
14528 let registry = Arc::new(Registry::default());
14529 let forwarding = Arc::new(ForwardingTable::default());
14530 let handler =
14531 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14532 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
14534 let client_connection = ConnectionId::new(202);
14535
14536 let responses = handler
14539 .handle_control(
14540 client_connection,
14541 hello_frame("aft-second", PROTOCOL_VERSION, 9),
14542 )
14543 .unwrap();
14544 assert_eq!(responses[0].header.ty, FrameType::Error);
14545 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
14546 assert!(registry.get_module("aft-second").unwrap().is_none());
14547 }
14548
14549 #[tokio::test]
14550 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
14551 let registry = Arc::new(Registry::default());
14552 let forwarding = Arc::new(ForwardingTable::default());
14553 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
14554 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
14555 hello_via_sink(
14556 &handler,
14557 &module_ctx,
14558 &mut module_rx,
14559 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
14560 )
14561 .await;
14562 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
14563 let pending = forwarding
14564 .begin_route_bind_relay_for_test(
14565 client_ctx.connection_id,
14566 client_ctx.egress.clone(),
14567 2,
14568 "alpha",
14569 )
14570 .unwrap();
14571 forwarding
14572 .complete_pending_relay(
14573 module_ctx.connection_id,
14574 pending.corr,
14575 RouteBindRelayOutcome::Accepted,
14576 )
14577 .unwrap();
14578 client_rx.try_recv().unwrap();
14579 for module_id in ["beta", "alpha"] {
14580 let replies = handler
14581 .handle_control_frame(
14582 &module_ctx,
14583 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
14584 )
14585 .await
14586 .unwrap();
14587 assert_eq!(replies.len(), 1, "second HELLO must be refused");
14588 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
14589 }
14590 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
14591 assert!(registry.get_module("beta").unwrap().is_none());
14592 assert!(matches!(
14593 forwarding
14594 .lookup_data_route(
14595 client_ctx.connection_id,
14596 pending.client_channel,
14597 pending.client_epoch,
14598 )
14599 .unwrap(),
14600 DataRoute::Client(DataRouteState::Bound(_))
14601 ));
14602 assert!(handler
14603 .hello_launch_nonces
14604 .lock()
14605 .unwrap()
14606 .presented(module_ctx.connection_id, Some("alpha-nonce")));
14607 assert!(module_rx.try_recv().is_err());
14608 }
14609
14610 #[test]
14611 fn reserved_module_hello_requires_matching_launch_nonce() {
14612 let registry = Arc::new(Registry::default());
14613 let supervisor = SupervisorHandle::new();
14614 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
14617 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14618
14619 let no_nonce = handler
14621 .handle_control(
14622 ConnectionId::new(1),
14623 hello_frame("vault", PROTOCOL_VERSION, 1),
14624 )
14625 .unwrap();
14626 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
14627 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
14628 assert!(registry.get_module("vault").unwrap().is_none());
14629
14630 let wrong = handler
14632 .handle_control(
14633 ConnectionId::new(2),
14634 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
14635 )
14636 .unwrap();
14637 assert_eq!(wrong[0].header.ty, FrameType::Error);
14638 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
14639 assert!(registry.get_module("vault").unwrap().is_none());
14640
14641 let ok = handler
14643 .handle_control(
14644 ConnectionId::new(3),
14645 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
14646 )
14647 .unwrap();
14648 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
14649 assert!(registry.get_module("vault").unwrap().is_some());
14650 }
14651
14652 #[test]
14653 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
14654 let registry = Arc::new(Registry::default());
14655 let supervisor = SupervisorHandle::new();
14656 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14657 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14658 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14659
14660 let squat = handler
14661 .handle_control(
14662 ConnectionId::new(1),
14663 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
14664 )
14665 .unwrap();
14666 assert_eq!(squat[0].header.ty, FrameType::Error);
14667 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
14668 assert!(parse_error(&squat[0])["message"]
14669 .as_str()
14670 .unwrap()
14671 .contains("fed:"));
14672
14673 let accepted_peer = handler
14674 .handle_control(
14675 ConnectionId::new(2),
14676 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
14677 )
14678 .unwrap();
14679 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
14680
14681 let accepted_short = handler
14682 .handle_control(
14683 ConnectionId::new(3),
14684 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
14685 )
14686 .unwrap();
14687 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
14688
14689 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
14690 let response = handler
14691 .handle_control(
14692 ConnectionId::new(conn),
14693 hello_frame(module_id, PROTOCOL_VERSION, conn),
14694 )
14695 .unwrap();
14696 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
14697 }
14698 }
14699
14700 #[test]
14701 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
14702 let registry = Arc::new(Registry::default());
14703 let supervisor = SupervisorHandle::new();
14704 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
14705 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
14706 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
14707 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
14708
14709 let owner_nonce = handler
14710 .handle_control(
14711 ConnectionId::new(1),
14712 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
14713 )
14714 .unwrap();
14715 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
14716 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
14717 assert!(registry.get_module("fed:special").unwrap().is_none());
14718
14719 let exact_nonce = handler
14720 .handle_control(
14721 ConnectionId::new(2),
14722 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
14723 )
14724 .unwrap();
14725 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
14726 assert!(registry.get_module("fed:special").unwrap().is_some());
14727 }
14728
14729 #[test]
14730 fn non_reserved_module_ignores_launch_nonce() {
14731 let registry = Arc::new(Registry::default());
14732 let handler = ControlHandler::new(Arc::clone(®istry));
14735 let no_nonce = handler
14736 .handle_control(
14737 ConnectionId::new(1),
14738 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
14739 )
14740 .unwrap();
14741 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
14742 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
14743
14744 let echoed_nonce = handler
14745 .handle_control(
14746 ConnectionId::new(2),
14747 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
14748 )
14749 .unwrap();
14750 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
14751 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
14752 }
14753
14754 #[test]
14755 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
14756 let handler = ControlHandler::default();
14757 let conn = ConnectionId::new(1);
14758 let malformed = Frame::build(
14759 FrameType::Hello,
14760 control_flags(),
14761 0,
14762 0,
14763 3,
14764 b"{not json".to_vec(),
14765 )
14766 .unwrap();
14767
14768 let error = handler.handle_control(conn, malformed).unwrap();
14769 assert_eq!(error[0].header.ty, FrameType::Error);
14770 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
14771
14772 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
14773 let pong = handler.handle_control(conn, ping).unwrap();
14774 assert_eq!(pong[0].header.ty, FrameType::Pong);
14775 assert_eq!(pong[0].header.corr, 4);
14776 }
14777
14778 #[test]
14779 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
14780 let registry = Arc::new(Registry::default());
14781 let handler = ControlHandler::new(Arc::clone(®istry));
14782
14783 handler
14784 .handle_control(
14785 ConnectionId::new(1),
14786 hello_frame("aft", PROTOCOL_VERSION, 1),
14787 )
14788 .unwrap();
14789 let duplicate = handler
14790 .handle_control(
14791 ConnectionId::new(2),
14792 hello_frame("aft", PROTOCOL_VERSION, 2),
14793 )
14794 .unwrap();
14795
14796 assert_eq!(duplicate[0].header.ty, FrameType::Error);
14797 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
14798 let registration = registry.get_module("aft").unwrap().unwrap();
14799 assert_eq!(registration.connection_id, ConnectionId::new(1));
14800 }
14801
14802 #[test]
14803 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
14804 let registry = Arc::new(Registry::default());
14805 let forwarding = Arc::new(ForwardingTable::default());
14806 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
14807 let handler =
14808 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14809 .with_process_liveness(process_liveness);
14810 let (ctx, route_channel, route_epoch) =
14811 bind_liveness_route(®istry, &forwarding, "aft-dead");
14812 let responses = handler
14813 .handle_route_poll(
14814 &ctx,
14815 route_poll_frame(41, PollKind::Liveness, route_channel),
14816 route_channel,
14817 route_epoch,
14818 PollKind::Liveness,
14819 )
14820 .unwrap();
14821
14822 assert_eq!(responses.len(), 1);
14823 assert_eq!(responses[0].header.ty, FrameType::Response);
14824 assert_route_poll_liveness(&responses[0], false);
14825 }
14826
14827 #[test]
14828 fn liveness_poll_without_process_source_uses_bound_route() {
14829 let registry = Arc::new(Registry::default());
14830 let forwarding = Arc::new(ForwardingTable::default());
14831 let handler =
14832 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14833 let (ctx, route_channel, route_epoch) =
14834 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
14835 let responses = handler
14836 .handle_route_poll(
14837 &ctx,
14838 route_poll_frame(42, PollKind::Liveness, route_channel),
14839 route_channel,
14840 route_epoch,
14841 PollKind::Liveness,
14842 )
14843 .unwrap();
14844
14845 assert_route_poll_liveness(&responses[0], true);
14846 }
14847
14848 #[test]
14849 fn liveness_poll_untracked_process_source_uses_bound_route() {
14850 let registry = Arc::new(Registry::default());
14851 let forwarding = Arc::new(ForwardingTable::default());
14852 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
14853 let handler =
14854 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
14855 .with_process_liveness(process_liveness);
14856 let (ctx, route_channel, route_epoch) =
14857 bind_liveness_route(®istry, &forwarding, "aft-untracked");
14858 let responses = handler
14859 .handle_route_poll(
14860 &ctx,
14861 route_poll_frame(43, PollKind::Liveness, route_channel),
14862 route_channel,
14863 route_epoch,
14864 PollKind::Liveness,
14865 )
14866 .unwrap();
14867
14868 assert_route_poll_liveness(&responses[0], true);
14869 }
14870
14871 #[tokio::test]
14872 async fn unknown_op_returns_unknown_control_op() {
14873 let handler = ControlHandler::default();
14874 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
14875 let request = Frame::build(
14876 FrameType::Request,
14877 control_flags(),
14878 0,
14879 0,
14880 55,
14881 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
14882 )
14883 .unwrap();
14884
14885 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14886
14887 assert_eq!(response.len(), 1);
14888 assert_eq!(response[0].header.ty, FrameType::Error);
14889 assert_eq!(response[0].header.corr, 55);
14890 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
14891 }
14892
14893 #[tokio::test]
14894 async fn supervisor_provenance_rejects_unknown_exact_module() {
14895 let handler = ControlHandler::default();
14896 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
14897 let request = Frame::build(
14898 FrameType::Request,
14899 control_flags(),
14900 0,
14901 0,
14902 57,
14903 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
14904 )
14905 .unwrap();
14906
14907 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
14908
14909 assert_eq!(response.len(), 1);
14910 assert_eq!(response[0].header.ty, FrameType::Error);
14911 assert_eq!(response[0].header.corr, 57);
14912 let error = parse_error(&response[0]);
14913 assert_eq!(error["code"], "unknown_module");
14914 assert_eq!(error["message"], "module_id 'missing' is not supervised");
14915 }
14916
14917 #[test]
14918 fn provenance_probe_override_keeps_handler_tests_deterministic() {
14919 let expected = subc_control::RunningImageAgreement::Unavailable {
14920 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14921 };
14922 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
14923 assert_eq!(handler.provenance_probe_override, Some(expected));
14924 }
14925
14926 #[test]
14927 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
14928 let verdict = reload_verdict(
14929 std::path::Path::new("/bin/new"),
14930 Some(std::path::Path::new("/bin/old")),
14931 subc_control::RunningImageAgreement::Unavailable {
14932 reason: subc_control::RunningImageUnavailableReason::HashFailed,
14933 },
14934 );
14935 assert!(matches!(
14936 verdict.path,
14937 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
14938 if configured == std::path::Path::new("/bin/new")
14939 && spawned_from == std::path::Path::new("/bin/old")
14940 ));
14941 }
14942
14943 #[test]
14944 fn launch_spec_without_running_digest_is_unavailable_not_match() {
14945 use subc_control::{
14946 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
14947 };
14948 let configured = crate::supervise::ModuleSpec {
14949 module_id: "legacy".into(),
14950 program: "/module".into(),
14951 args: Vec::new(),
14952 env: Vec::new(),
14953 reserved: false,
14954 reserved_prefixes: Vec::new(),
14955 protocol: subc_control::ModuleProtocol::Subc,
14956 overlap: Default::default(),
14957 };
14958 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
14959 assert_eq!(
14960 launch_spec_agreement(&configured, false, Some(&digest)),
14961 Agreement::Unavailable {
14962 reason: Reason::NotRunning
14963 }
14964 );
14965 assert_eq!(
14966 launch_spec_agreement(&configured, true, None),
14967 Agreement::Unavailable {
14968 reason: Reason::SpawnedSpecUnavailable
14969 }
14970 );
14971 assert_eq!(
14972 launch_spec_agreement(&configured, true, Some(&digest)),
14973 Agreement::Match
14974 );
14975 }
14976
14977 #[cfg(windows)]
14978 #[test]
14979 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
14980 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
14981 let configured = dir.join("module.exe");
14982 std::fs::write(&configured, b"executable path fixture").unwrap();
14983 let spawned = std::fs::canonicalize(&configured).unwrap();
14984 assert_ne!(
14985 configured, spawned,
14986 "fixture must exercise different path spellings"
14987 );
14988 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
14989 let lower_case = configured.to_string_lossy().to_lowercase();
14990 for spelling in [
14991 &configured,
14992 Path::new(&forward_slashes),
14993 Path::new(&lower_case),
14994 ] {
14995 let verdict = reload_verdict(
14996 spelling,
14997 Some(&spawned),
14998 subc_control::RunningImageAgreement::Unavailable {
14999 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15000 },
15001 );
15002 assert_eq!(
15003 verdict.path,
15004 ReloadPathAgreement::Match,
15005 "configured {spelling:?}, spawned {spawned:?}"
15006 );
15007 }
15008 }
15009
15010 #[cfg(windows)]
15011 #[test]
15012 fn windows_reload_path_keeps_different_existing_program_pending() {
15013 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
15014 let old = dir.join("old.exe");
15015 let new = dir.join("new.exe");
15016 std::fs::write(&old, b"same executable content").unwrap();
15017 std::fs::copy(&old, &new).unwrap();
15018 let spawned = std::fs::canonicalize(&old).unwrap();
15019 let image = subc_control::RunningImageAgreement::Unavailable {
15020 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15021 };
15022 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
15023 assert!(
15024 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
15025 if configured == new && spawned_from == spawned)
15026 );
15027 let missing = dir.join("missing.exe");
15028 assert!(matches!(
15029 reload_verdict(&missing, Some(&spawned), image).path,
15030 ReloadPathAgreement::Mismatch { .. }
15031 ));
15032 }
15033
15034 #[test]
15035 fn reload_verdict_detects_replaced_image_at_same_path() {
15036 let image = subc_control::RunningImageAgreement::Mismatch {
15037 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
15038 digest: "old".into(),
15039 },
15040 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
15041 digest: "new".into(),
15042 },
15043 };
15044 let verdict = reload_verdict(
15045 std::path::Path::new("/bin/same"),
15046 Some(std::path::Path::new("/bin/same")),
15047 image.clone(),
15048 );
15049 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15050 assert_eq!(verdict.image, image);
15051 }
15052
15053 #[test]
15054 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
15055 let image = subc_control::RunningImageAgreement::Unavailable {
15056 reason: subc_control::RunningImageUnavailableReason::NotRunning,
15057 };
15058 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
15059 assert_eq!(
15060 verdict.path,
15061 subc_control::ReloadPathAgreement::Unavailable {
15062 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
15063 }
15064 );
15065 assert_eq!(verdict.image, image);
15066
15067 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
15068 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
15069 };
15070 let verdict = reload_verdict(
15071 std::path::Path::new("/bin/same"),
15072 Some(std::path::Path::new("/bin/same")),
15073 unconfirmed.clone(),
15074 );
15075 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15076 assert_eq!(verdict.image, unconfirmed);
15077 }
15078
15079 #[test]
15080 fn reload_verdict_preserves_each_image_unavailability_reason() {
15081 use subc_control::RunningImageUnavailableReason as Reason;
15082
15083 for reason in [
15084 Reason::NotRunning,
15085 Reason::UnsupportedPlatform,
15086 Reason::RunningExecutableUnreadable,
15087 Reason::SpawnedPathUnreadable,
15088 Reason::HashFailed,
15089 Reason::ProcessIdentityUnconfirmed,
15090 Reason::Unknown("future_probe_reason".to_string()),
15091 ] {
15092 let image = subc_control::RunningImageAgreement::Unavailable {
15093 reason: reason.clone(),
15094 };
15095 let verdict = reload_verdict(
15096 std::path::Path::new("/bin/same"),
15097 Some(std::path::Path::new("/bin/same")),
15098 image.clone(),
15099 );
15100 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
15101 assert_eq!(verdict.image, image, "{reason:?}");
15102 }
15103 }
15104
15105 #[tokio::test]
15106 async fn malformed_control_bodies_return_invalid_control_body() {
15107 let handler = ControlHandler::default();
15108 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
15109
15110 for (corr, body) in [
15111 (56, br#"{"route_channel":1}"#.as_slice()),
15112 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
15113 (
15114 58,
15115 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
15116 ),
15117 ] {
15118 let request = Frame::build(
15119 FrameType::Request,
15120 control_flags(),
15121 0,
15122 0,
15123 corr,
15124 body.to_vec(),
15125 )
15126 .unwrap();
15127 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15128
15129 assert_eq!(response.len(), 1);
15130 assert_eq!(response[0].header.ty, FrameType::Error);
15131 assert_eq!(response[0].header.corr, corr);
15132 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
15133 }
15134 }
15135
15136 #[tokio::test]
15137 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
15138 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15139 let registry = Arc::new(Registry::default());
15140 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
15141 let router = Router::with_control_handler(Arc::clone(&control));
15142 let connection = router.begin_connection();
15143 let (ctx, mut rx) = route_ctx(connection.id());
15144
15145 router
15146 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
15147 .await
15148 .unwrap();
15149 let response = rx.recv().await.unwrap();
15150 let ack = parse_ack(&response);
15151 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15152 let channel = 1;
15153
15154 let goodbye =
15155 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
15156 router.route_for_connection(&ctx, goodbye).await.unwrap();
15157 assert!(rx.try_recv().is_err());
15158 assert!(registry.get_module("aft").unwrap().is_none());
15159
15160 router
15161 .route_for_connection(&ctx, channel_request(channel, 13))
15162 .await
15163 .unwrap();
15164 let error_frame = rx.recv().await.unwrap();
15165 assert_eq!(error_frame.header.ty, FrameType::Error);
15166 assert_eq!(error_frame.header.channel, channel);
15167 let captured = crate::router::test_log::captured_logs(&logs);
15168 assert_eq!(
15169 captured
15170 .lines()
15171 .filter(|line| {
15172 line.contains("module_id=aft")
15173 && line.contains("reason=explicit_goodbye")
15174 && line.contains("module registration ended")
15175 })
15176 .count(),
15177 1,
15178 "unexpected GOODBYE registry log: {captured}"
15179 );
15180 }
15181
15182 #[tokio::test]
15183 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
15184 let registry = Arc::new(Registry::default());
15185 let handler = ControlHandler::new(registry).with_capability_config(
15186 [("prov".to_string(), true), ("cons".to_string(), true)],
15187 BTreeMap::new(),
15188 );
15189 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
15190 register_capability_manifest(
15191 &handler,
15192 &provider_ctx,
15193 &mut provider_rx,
15194 capability_manifest("prov", &["thing/v1"], &[]),
15195 1,
15196 )
15197 .await;
15198 let mut consumer = capability_manifest("cons", &[], &[]);
15199 consumer.capabilities.as_mut().unwrap().requires.push(
15200 subc_protocol::manifest::CapabilityRequirement {
15201 capability: "thing/v1".to_string(),
15202 need: subc_protocol::manifest::CapabilityNeed::Required,
15203 },
15204 );
15205 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
15206 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
15207 assert_eq!(
15208 handler.capability_evaluator.verdict("cons", "thing/v1"),
15209 Some(CapabilityVerdict::Provided)
15210 );
15211 let (mut client_rx, _) = open_route_for_capability_test(
15212 &handler,
15213 &provider_ctx,
15214 &mut provider_rx,
15215 703,
15216 3,
15217 "prov",
15218 None,
15219 )
15220 .await;
15221 let goodbye =
15222 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15223 handler
15224 .handle_control_frame(&provider_ctx, goodbye)
15225 .await
15226 .unwrap();
15227 assert_eq!(
15228 handler.capability_evaluator.verdict("cons", "thing/v1"),
15229 Some(CapabilityVerdict::NeverProvided)
15230 );
15231 let closed = client_rx
15232 .try_recv()
15233 .expect("GOODBYE pushes route.closed before route GOODBYE");
15234 assert!(
15235 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
15236 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
15237 );
15238 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
15239 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
15240 }
15241
15242 struct RouteClosedFixture {
15243 handler: ControlHandler,
15244 #[cfg(unix)]
15245 supervisor: Supervisor,
15246 module_ctx: RouteCtx,
15247 #[cfg(unix)]
15248 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15249 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
15250 route: Arc<crate::forwarding::RouteBinding>,
15251 }
15252
15253 async fn route_closed_fixture() -> RouteClosedFixture {
15254 let registry = Arc::new(Registry::default());
15255 let handler = ControlHandler::new(Arc::clone(®istry));
15256 #[cfg(unix)]
15257 let supervisor = Supervisor::new(registry, RestartPolicy::default())
15258 .with_forwarding(Arc::clone(&handler.forwarding));
15259 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
15260 register_capability_manifest(
15261 &handler,
15262 &module_ctx,
15263 &mut module_rx,
15264 capability_manifest("prov", &[], &[]),
15265 1,
15266 )
15267 .await;
15268 let (client_rx, bind) = open_route_for_capability_test(
15269 &handler,
15270 &module_ctx,
15271 &mut module_rx,
15272 702,
15273 2,
15274 "prov",
15275 None,
15276 )
15277 .await;
15278 let ModuleControlRequest::RouteBind {
15279 route_channel: channel,
15280 epoch: route_epoch,
15281 ..
15282 } = bind
15283 else {
15284 panic!("expected route.bind");
15285 };
15286 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
15287 handler
15288 .forwarding
15289 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
15290 .unwrap()
15291 else {
15292 panic!("the acknowledged route must be bound");
15293 };
15294 RouteClosedFixture {
15295 handler,
15296 #[cfg(unix)]
15297 supervisor,
15298 module_ctx,
15299 #[cfg(unix)]
15300 module_rx,
15301 client_rx,
15302 route,
15303 }
15304 }
15305
15306 impl RouteClosedFixture {
15307 #[cfg(unix)]
15308 async fn cut(&mut self) {
15309 let notices = async {
15310 let mut draining = self.module_rx.recv().await.unwrap();
15311 assert!(matches!(
15312 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
15313 &draining.body
15314 )
15315 .unwrap(),
15316 subc_protocol::session::ModuleControlCommand::Draining {
15317 reason: RouteCloseReason::Restart,
15318 ..
15319 }
15320 ));
15321 draining.flushed.take().unwrap().send(()).unwrap();
15322 let mut closing = self.client_rx.recv().await.unwrap();
15323 assert!(matches!(
15324 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
15325 ClientControlPush::RouteClosing {
15326 reason: RouteCloseReason::Restart,
15327 ..
15328 }
15329 ));
15330 closing.flushed.take().unwrap().send(()).unwrap();
15331 };
15332 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
15333 result.unwrap();
15334 }
15335
15336 fn disconnect(&mut self) -> ClientControlPush {
15337 self.handler
15338 .cleanup_connection(self.module_ctx.connection_id)
15339 .unwrap();
15340 let closed = self.client_rx.try_recv().unwrap();
15341 assert_eq!(closed.header.ty, FrameType::Push);
15342 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
15343 assert!(matches!(
15344 &push,
15345 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
15346 if channels == &[self.route.client_channel]
15347 ));
15348 assert_eq!(
15349 self.client_rx.try_recv().unwrap().header.ty,
15350 FrameType::Goodbye
15351 );
15352 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
15353 push
15354 }
15355 }
15356
15357 #[cfg(unix)]
15358 #[tokio::test(start_paused = true)]
15359 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
15360 let mut fixture = route_closed_fixture().await;
15361 fixture.cut().await;
15362 assert!(matches!(
15363 fixture.disconnect(),
15364 ClientControlPush::RouteClosed {
15365 reason: RouteCloseReason::Restart,
15366 drained: true,
15367 ..
15368 }
15369 ));
15370 }
15371
15372 #[cfg(unix)]
15373 #[tokio::test(start_paused = true)]
15374 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
15375 let mut fixture = route_closed_fixture().await;
15376 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
15377 let started = tokio::time::Instant::now();
15378 fixture.cut().await;
15379 assert!(started.elapsed() >= Duration::from_secs(2));
15380 assert_eq!(fixture.route.flow.in_flight(), 1);
15381 assert!(matches!(
15382 fixture.disconnect(),
15383 ClientControlPush::RouteClosed {
15384 reason: RouteCloseReason::Restart,
15385 drained: false,
15386 ..
15387 }
15388 ));
15389 }
15390
15391 #[tokio::test]
15392 async fn module_connection_crash_reports_crash_and_not_drained() {
15393 let mut fixture = route_closed_fixture().await;
15394 assert!(matches!(
15398 fixture.disconnect(),
15399 ClientControlPush::RouteClosed {
15400 reason: RouteCloseReason::Crash,
15401 drained: false,
15402 ..
15403 }
15404 ));
15405 }
15406
15407 #[tokio::test]
15408 async fn dropping_router_connection_releases_registration() {
15409 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15410 let registry = Arc::new(Registry::default());
15411 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
15412 let router = Router::with_control_handler(Arc::clone(&control));
15413 let connection = router.begin_connection();
15414 let connection_id = connection.id();
15415 let (ctx, mut rx) = route_ctx(connection_id);
15416
15417 router
15418 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
15419 .await
15420 .unwrap();
15421 let response = rx.recv().await.unwrap();
15422 let ack = parse_ack(&response);
15423 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
15424 assert!(registry.get_module("aft").unwrap().is_some());
15425
15426 drop(connection);
15427
15428 assert!(registry.get_module("aft").unwrap().is_none());
15429 assert_eq!(registry.active_registration_count().unwrap(), 0);
15430
15431 control
15432 .cleanup_connection(ConnectionId::new(u64::MAX))
15433 .unwrap();
15434 let captured = crate::router::test_log::captured_logs(&logs);
15435 println!("captured registry lifecycle logs:\n{captured}");
15436 let events: Vec<_> = captured
15437 .lines()
15438 .filter(|line| {
15439 line.contains("module registered module_id=aft ")
15440 || line.contains("module registration ended")
15441 })
15442 .collect();
15443 assert_eq!(
15444 events.len(),
15445 2,
15446 "unexpected registry lifecycle logs: {captured}"
15447 );
15448 assert!(events[0].contains("module registered module_id=aft "));
15449 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
15450 assert!(events[1].contains(&format!(
15451 "module_id=aft connection_id={}",
15452 connection_id.get()
15453 )));
15454 assert!(events[1].contains("reason=connection_closed"));
15455 assert!(events[1].contains("module registration ended"));
15456 assert_eq!(
15457 captured
15458 .lines()
15459 .filter(|line| line.contains("module registered module_id=aft "))
15460 .count(),
15461 1,
15462 "legacy registration admission line must appear once: {captured}"
15463 );
15464 }
15465
15466 #[tokio::test]
15467 async fn hello_registration_keeps_legacy_module_registered_line_once() {
15468 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
15469 let registry = Arc::new(Registry::default());
15470 let control = ControlHandler::new(Arc::clone(®istry));
15471 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
15472 hello_via_sink(
15473 &control,
15474 &ctx,
15475 &mut rx,
15476 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
15477 )
15478 .await;
15479
15480 let captured = crate::router::test_log::captured_logs(&logs);
15481 let admissions: Vec<_> = captured
15482 .lines()
15483 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
15484 .collect();
15485 assert_eq!(
15486 admissions.len(),
15487 1,
15488 "legacy admission line must remain exactly once: {captured}"
15489 );
15490 assert!(admissions[0].contains("routable_provider=true"));
15491 assert!(admissions[0].contains("connection_id=777"));
15492 }
15493
15494 fn capability_manifest(
15495 module_id: &str,
15496 provides: &[&str],
15497 must_never_reach: &[&str],
15498 ) -> ModuleManifest {
15499 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
15500 manifest.capabilities = Some(CapabilityDeclarations {
15501 provides: provides
15502 .iter()
15503 .map(|capability| (*capability).to_string())
15504 .collect(),
15505 requires: Vec::new(),
15506 must_never_reach: must_never_reach
15507 .iter()
15508 .map(|capability| (*capability).to_string())
15509 .collect(),
15510 });
15511 manifest
15512 }
15513
15514 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
15515 Frame::build(
15516 FrameType::Hello,
15517 control_flags(),
15518 0,
15519 0,
15520 corr,
15521 serde_json::to_vec(&ModuleHelloBody {
15522 protocol_ver: manifest.protocol_ver,
15523 manifest,
15524 control_ops: None,
15525 launch_nonce: None,
15526 })
15527 .expect("capability test HELLO serializes"),
15528 )
15529 .expect("capability test HELLO frame builds")
15530 }
15531
15532 fn catalog_update_with_capabilities_frame(
15533 corr: u64,
15534 capabilities: CapabilityDeclarations,
15535 ) -> Frame {
15536 Frame::build(
15537 FrameType::Request,
15538 control_flags(),
15539 0,
15540 0,
15541 corr,
15542 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
15543 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
15544 capabilities: Some(capabilities),
15545 ready: None,
15546 })
15547 .expect("capability catalog.update serializes"),
15548 )
15549 .expect("capability catalog.update frame builds")
15550 }
15551
15552 async fn register_capability_manifest(
15553 handler: &ControlHandler,
15554 ctx: &RouteCtx,
15555 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15556 manifest: ModuleManifest,
15557 corr: u64,
15558 ) {
15559 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
15560 }
15561
15562 async fn open_route_for_capability_test(
15563 handler: &ControlHandler,
15564 target_ctx: &RouteCtx,
15565 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
15566 client_connection_id: u64,
15567 corr: u64,
15568 target_module_id: &str,
15569 consumer_identity: Option<ConsumerIdentity>,
15570 ) -> (
15571 mpsc::Receiver<crate::router::OutboundFrame>,
15572 ModuleControlRequest,
15573 ) {
15574 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
15575 let route_handler = handler.clone();
15576 let target_module_id = target_module_id.to_string();
15577 let route_task = tokio::spawn(async move {
15578 route_handler
15579 .handle_control_frame(
15580 &client_ctx,
15581 route_open_frame_with_admission_facts(
15582 corr,
15583 &target_module_id,
15584 unique_project_root("admission-facts"),
15585 consumer_identity,
15586 None,
15587 ),
15588 )
15589 .await
15590 .expect("capability test route.open succeeds")
15591 });
15592 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
15593 .await
15594 .expect("capability test route.open must reach route.bind")
15595 .expect("target control receiver stays open");
15596 let bind_request: ModuleControlRequest =
15597 serde_json::from_slice(&bind.body).expect("route.bind decodes");
15598 handler
15599 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
15600 .await
15601 .expect("capability test route.bind ACK succeeds");
15602 assert!(route_task.await.expect("route.open task joins").is_empty());
15603 let opened = client_rx
15604 .recv()
15605 .await
15606 .expect("successful route.open publishes a response");
15607 assert!(matches!(
15608 serde_json::from_slice::<ClientControlResponse>(&opened.body),
15609 Ok(ClientControlResponse::RouteOpen { .. })
15610 ));
15611 (client_rx, bind_request)
15612 }
15613
15614 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
15615 assert_eq!(frame.header.ty, FrameType::Push);
15616 assert_eq!(frame.header.channel, 0);
15617 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
15618 .expect("route.closed control push decodes");
15619 let ClientControlPush::RouteClosed { channels, .. } = &push else {
15620 panic!("expected route.closed");
15621 };
15622 assert_eq!(channels.len(), 1, "exactly one violating route closed");
15623 let closed_channel = channels[0];
15624 let channels = channels.clone();
15625 assert_eq!(
15626 push,
15627 ClientControlPush::RouteClosed {
15628 module_id: target_module_id.to_string(),
15629 channels,
15630 reason: RouteCloseReason::CapabilityDenied,
15631 drained: false,
15632 abandoned: 0,
15633 excluded_subscriptions: 0,
15634 terminal: Some(false),
15635 }
15636 );
15637 closed_channel
15638 }
15639
15640 #[tokio::test]
15641 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
15642 let registry = Arc::new(Registry::default());
15643 let forwarding = Arc::new(ForwardingTable::default());
15644 let supervisor = SupervisorHandle::new();
15645 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15646 let handler =
15647 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15648 .with_supervisor(supervisor);
15649 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
15650 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
15651 register_capability_manifest(
15652 &handler,
15653 &target_ctx,
15654 &mut target_rx,
15655 capability_manifest("target", &["credentials-provider/v1"], &[]),
15656 1,
15657 )
15658 .await;
15659 register_capability_manifest(
15660 &handler,
15661 &opener_ctx,
15662 &mut opener_rx,
15663 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15664 2,
15665 )
15666 .await;
15667
15668 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
15669 let replies = handler
15670 .handle_control_frame(
15671 &client_ctx,
15672 route_open_frame_with_admission_facts(
15673 3,
15674 "target",
15675 unique_project_root("admission-facts"),
15676 Some(ConsumerIdentity {
15677 module_id: "opener".to_string(),
15678 launch_nonce: "opener-nonce".to_string(),
15679 }),
15680 None,
15681 ),
15682 )
15683 .await
15684 .expect("denied route.open returns a typed frame");
15685 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
15686 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15687 assert!(
15688 target_rx.try_recv().is_err(),
15689 "forbidden route.open must not relay route.bind"
15690 );
15691 }
15692
15693 #[tokio::test]
15694 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
15695 let registry = Arc::new(Registry::default());
15696 let forwarding = Arc::new(ForwardingTable::default());
15697 let supervisor = SupervisorHandle::new();
15698 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15699 let handler =
15700 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15701 .with_supervisor(supervisor);
15702 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
15703 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
15704 register_capability_manifest(
15705 &handler,
15706 &target_ctx,
15707 &mut target_rx,
15708 capability_manifest("target", &["credentials-provider/v1"], &[]),
15709 1,
15710 )
15711 .await;
15712 register_capability_manifest(
15713 &handler,
15714 &old_opener_ctx,
15715 &mut old_opener_rx,
15716 capability_manifest("opener", &[], &[]),
15717 2,
15718 )
15719 .await;
15720 let (mut client_rx, _) = open_route_for_capability_test(
15721 &handler,
15722 &target_ctx,
15723 &mut target_rx,
15724 712,
15725 3,
15726 "target",
15727 Some(ConsumerIdentity {
15728 module_id: "opener".to_string(),
15729 launch_nonce: "opener-nonce".to_string(),
15730 }),
15731 )
15732 .await;
15733 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15734
15735 handler
15736 .cleanup_connection(old_opener_ctx.connection_id)
15737 .expect("old opener registration cleans up");
15738 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
15739 register_capability_manifest(
15740 &handler,
15741 &new_opener_ctx,
15742 &mut new_opener_rx,
15743 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15744 4,
15745 )
15746 .await;
15747
15748 let closed_channel = assert_capability_denied_push(
15749 client_rx
15750 .try_recv()
15751 .expect("HELLO deny addition must emit route.closed")
15752 .frame,
15753 "target",
15754 );
15755 let goodbye = client_rx
15756 .try_recv()
15757 .expect("HELLO deny addition must send the client a route GOODBYE")
15758 .frame;
15759 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15760 assert_eq!(goodbye.header.channel, closed_channel);
15761 assert_ne!(goodbye.header.channel, 0);
15762 assert_ne!(goodbye.header.epoch, 0);
15763 assert!(client_rx.try_recv().is_err());
15764 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15765 assert!(matches!(
15766 target_rx.try_recv(),
15767 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15768 ));
15769 }
15770
15771 #[tokio::test]
15772 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
15773 let registry = Arc::new(Registry::default());
15774 let forwarding = Arc::new(ForwardingTable::default());
15775 let supervisor = SupervisorHandle::new();
15776 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15777 let handler =
15778 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15779 .with_supervisor(supervisor);
15780 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
15781 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
15782 register_capability_manifest(
15783 &handler,
15784 &target_ctx,
15785 &mut target_rx,
15786 capability_manifest("target", &[], &[]),
15787 1,
15788 )
15789 .await;
15790 register_capability_manifest(
15791 &handler,
15792 &opener_ctx,
15793 &mut opener_rx,
15794 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15795 2,
15796 )
15797 .await;
15798 let (mut client_rx, _) = open_route_for_capability_test(
15799 &handler,
15800 &target_ctx,
15801 &mut target_rx,
15802 722,
15803 3,
15804 "target",
15805 Some(ConsumerIdentity {
15806 module_id: "opener".to_string(),
15807 launch_nonce: "opener-nonce".to_string(),
15808 }),
15809 )
15810 .await;
15811 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15812
15813 let replies = handler
15814 .handle_control_frame(
15815 &target_ctx,
15816 catalog_update_with_capabilities_frame(
15817 4,
15818 CapabilityDeclarations {
15819 provides: vec!["credentials-provider/v1".to_string()],
15820 requires: Vec::new(),
15821 must_never_reach: Vec::new(),
15822 },
15823 ),
15824 )
15825 .await
15826 .expect("claim catalog.update succeeds");
15827 assert!(matches!(
15828 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
15829 Ok(ModuleControlResponseToModule::CatalogUpdate {})
15830 ));
15831 let closed_channel = assert_capability_denied_push(
15832 client_rx
15833 .try_recv()
15834 .expect("claim addition must emit route.closed")
15835 .frame,
15836 "target",
15837 );
15838 let goodbye = client_rx
15839 .try_recv()
15840 .expect("claim addition must send the client a route GOODBYE")
15841 .frame;
15842 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
15843 assert_eq!(goodbye.header.channel, closed_channel);
15844 assert_ne!(goodbye.header.channel, 0);
15845 assert_ne!(goodbye.header.epoch, 0);
15846 assert!(client_rx.try_recv().is_err());
15847 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
15848 assert!(matches!(
15849 target_rx.try_recv(),
15850 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
15851 ));
15852 }
15853
15854 #[tokio::test]
15855 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
15856 let registry = Arc::new(Registry::default());
15857 let forwarding = Arc::new(ForwardingTable::default());
15858 let supervisor = SupervisorHandle::new();
15859 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15860 let handler =
15861 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15862 .with_supervisor(supervisor);
15863 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
15864 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
15865 register_capability_manifest(
15866 &handler,
15867 &target_ctx,
15868 &mut target_rx,
15869 capability_manifest("target", &["credentials-provider/v1"], &[]),
15870 1,
15871 )
15872 .await;
15873 register_capability_manifest(
15874 &handler,
15875 &opener_ctx,
15876 &mut opener_rx,
15877 capability_manifest("opener", &[], &[]),
15878 2,
15879 )
15880 .await;
15881 let (mut client_rx, _) = open_route_for_capability_test(
15882 &handler,
15883 &target_ctx,
15884 &mut target_rx,
15885 732,
15886 3,
15887 "target",
15888 Some(ConsumerIdentity {
15889 module_id: "opener".to_string(),
15890 launch_nonce: "opener-nonce".to_string(),
15891 }),
15892 )
15893 .await;
15894 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15895
15896 handler
15897 .handle_control_frame(
15898 &target_ctx,
15899 catalog_update_with_capabilities_frame(
15900 4,
15901 CapabilityDeclarations {
15902 provides: Vec::new(),
15903 requires: Vec::new(),
15904 must_never_reach: Vec::new(),
15905 },
15906 ),
15907 )
15908 .await
15909 .expect("claim removal catalog.update succeeds");
15910 assert_eq!(
15911 forwarding.active_binding_count().unwrap(),
15912 1,
15913 "removing an attested target claim must leave the route census unchanged"
15914 );
15915 assert!(
15916 client_rx.try_recv().is_err(),
15917 "claim removal must not emit route.closed or a client route GOODBYE"
15918 );
15919 assert!(
15920 target_rx.try_recv().is_err(),
15921 "claim removal must not send the target a route GOODBYE"
15922 );
15923 }
15924
15925 #[tokio::test]
15928 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
15929 let registry = Arc::new(Registry::default());
15930 let forwarding = Arc::new(ForwardingTable::default());
15931 let supervisor = SupervisorHandle::new();
15932 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
15933 let handler =
15934 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15935 .with_supervisor(supervisor);
15936 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
15937 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
15938 register_capability_manifest(
15939 &handler,
15940 &target_ctx,
15941 &mut target_rx,
15942 capability_manifest("target", &["credentials-provider/v1"], &[]),
15943 1,
15944 )
15945 .await;
15946 register_capability_manifest(
15947 &handler,
15948 &opener_ctx,
15949 &mut opener_rx,
15950 capability_manifest("opener", &[], &["credentials-provider/v1"]),
15951 2,
15952 )
15953 .await;
15954
15955 let (_client_rx, bind) = open_route_for_capability_test(
15956 &handler,
15957 &target_ctx,
15958 &mut target_rx,
15959 742,
15960 3,
15961 "target",
15962 None,
15963 )
15964 .await;
15965 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
15966 panic!("direct scope-honesty route must bind");
15967 };
15968 assert_eq!(principal, Some(Principal::Direct));
15969 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
15970 }
15971
15972 #[tokio::test]
15975 async fn must_never_reach_self_route_is_capability_forbidden() {
15976 let registry = Arc::new(Registry::default());
15977 let forwarding = Arc::new(ForwardingTable::default());
15978 let supervisor = SupervisorHandle::new();
15979 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
15980 let handler =
15981 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15982 .with_supervisor(supervisor);
15983 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
15984 register_capability_manifest(
15985 &handler,
15986 &self_ctx,
15987 &mut self_rx,
15988 capability_manifest(
15989 "self-provider",
15990 &["credentials-provider/v1"],
15991 &["credentials-provider/v1"],
15992 ),
15993 1,
15994 )
15995 .await;
15996
15997 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
15998 let replies = handler
15999 .handle_control_frame(
16000 &client_ctx,
16001 route_open_frame_with_admission_facts(
16002 2,
16003 "self-provider",
16004 unique_project_root("admission-facts"),
16005 Some(ConsumerIdentity {
16006 module_id: "self-provider".to_string(),
16007 launch_nonce: "self-nonce".to_string(),
16008 }),
16009 None,
16010 ),
16011 )
16012 .await
16013 .expect("self-route refusal returns a typed frame");
16014 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16015 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16016 assert!(
16017 self_rx.try_recv().is_err(),
16018 "self denial must not relay route.bind"
16019 );
16020 }
16021
16022 #[test]
16023 fn unsupported_channel_zero_frame_returns_error() {
16024 let handler = ControlHandler::default();
16025 let request = Frame::build(
16026 FrameType::Request,
16027 control_flags(),
16028 0,
16029 0,
16030 21,
16031 b"opaque".to_vec(),
16032 )
16033 .unwrap();
16034
16035 let response = handler
16036 .handle_control(ConnectionId::new(1), request)
16037 .unwrap();
16038
16039 assert_eq!(response[0].header.ty, FrameType::Error);
16040 assert_eq!(
16041 parse_error(&response[0])["code"],
16042 "unsupported_control_frame"
16043 );
16044 }
16045
16046 mod swap {
16051 use super::*;
16052
16053 const INCUMBENT: ConnectionId = ConnectionId::new(30);
16054 const CANDIDATE: ConnectionId = ConnectionId::new(40);
16055
16056 struct Swap {
16057 registry: Arc<Registry>,
16058 forwarding: Arc<ForwardingTable>,
16059 handler: ControlHandler,
16060 incumbent_ctx: RouteCtx,
16061 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16062 candidate_ctx: RouteCtx,
16063 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16064 }
16065
16066 async fn swap_with_incumbent() -> Swap {
16067 let registry = Arc::new(Registry::default());
16068 let forwarding = Arc::new(ForwardingTable::default());
16069 let handler =
16070 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
16071 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
16072 hello_via_sink(
16073 &handler,
16074 &incumbent_ctx,
16075 &mut incumbent_rx,
16076 hello_frame("aft", PROTOCOL_VERSION, 7),
16077 )
16078 .await;
16079 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
16080 Swap {
16081 registry,
16082 forwarding,
16083 handler,
16084 incumbent_ctx,
16085 incumbent_rx,
16086 candidate_ctx,
16087 candidate_rx,
16088 }
16089 }
16090
16091 fn register_candidate(swap: &Swap, ready: Option<bool>) {
16092 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
16093 candidate_manifest.ready = ready;
16094 let registration = swap
16095 .registry
16096 .register_candidate_with_control_ops(
16097 candidate_manifest,
16098 PROTOCOL_VERSION,
16099 CANDIDATE,
16100 module_baseline_control_ops(),
16101 )
16102 .unwrap();
16103 swap.forwarding
16104 .register_candidate_module_connection(
16105 CANDIDATE,
16106 "aft".to_string(),
16107 PROTOCOL_VERSION,
16108 manifest_concurrency(®istration.manifest),
16109 swap.candidate_ctx.egress.clone(),
16110 )
16111 .unwrap();
16112 }
16113
16114 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
16115 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
16116 swap.registry.promote_candidate("aft").unwrap().unwrap();
16117 cutover.incumbent.unwrap()
16118 }
16119
16120 #[tokio::test]
16129 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
16130 let mut swap = swap_with_incumbent().await;
16131 let hold = Arc::new(std::sync::Barrier::new(2));
16132 let handler = swap
16133 .handler
16134 .clone()
16135 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
16136 handler
16137 .supervisor
16138 .open_swap("aft", "candidate-nonce".to_string());
16139 let hello = {
16140 let handler = handler.clone();
16141 let candidate_ctx = swap.candidate_ctx.clone();
16142 std::thread::spawn(move || {
16143 handler
16144 .handle_hello(
16145 candidate_ctx.connection_id,
16146 Some(candidate_ctx.egress.clone()),
16147 hello_frame_with_nonce(
16148 "aft",
16149 PROTOCOL_VERSION,
16150 8,
16151 Some("candidate-nonce"),
16152 ),
16153 )
16154 .unwrap()
16155 })
16156 };
16157
16158 hold.wait();
16159 assert!(swap
16162 .registry
16163 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
16164 .unwrap()
16165 .is_some());
16166 assert!(swap
16167 .registry
16168 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16169 .unwrap()
16170 .is_none());
16171 assert!(swap
16172 .forwarding
16173 .module_endpoint_for_connection(CANDIDATE)
16174 .unwrap()
16175 .is_none());
16176
16177 hold.wait();
16178 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
16179 assert_eq!(
16180 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
16181 FrameType::HelloAck
16182 );
16183 let published = swap
16184 .registry
16185 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
16186 .unwrap()
16187 .expect("candidate is published once its HELLO completes");
16188 assert_eq!(published.connection_id, CANDIDATE);
16189 assert!(swap
16190 .forwarding
16191 .module_endpoint_for_connection(CANDIDATE)
16192 .unwrap()
16193 .is_some());
16194 }
16195
16196 fn keyed_total(counters: &Value, key: &str) -> u64 {
16197 counters[key]
16198 .as_object()
16199 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
16200 .unwrap_or(0)
16201 }
16202
16203 #[tokio::test]
16204 async fn replacement_logs_old_end_and_new_admission_once() {
16205 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16206 let mut swap = swap_with_incumbent().await;
16207 swap.handler
16208 .supervisor
16209 .open_swap("aft", "candidate-nonce".to_string());
16210 hello_via_sink(
16211 &swap.handler,
16212 &swap.candidate_ctx,
16213 &mut swap.candidate_rx,
16214 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
16215 )
16216 .await;
16217 cutover(&swap);
16218 swap.handler.cleanup_connection(INCUMBENT).unwrap();
16219
16220 let captured = crate::router::test_log::captured_logs(&logs);
16221 println!("captured replacement registry logs:\n{captured}");
16222 let new_admissions: Vec<_> = captured
16223 .lines()
16224 .filter(|line| {
16225 line.contains("connection_id=40")
16226 && line.contains("swap candidate registered; not routable until cutover")
16227 })
16228 .collect();
16229 assert_eq!(
16230 new_admissions.len(),
16231 1,
16232 "unexpected admission logs: {captured}"
16233 );
16234 assert!(new_admissions[0].contains("module_id=aft"));
16235 assert!(new_admissions[0].contains("connection_id=40"));
16236 assert!(
16237 new_admissions[0].contains("swap candidate registered; not routable until cutover")
16238 );
16239 assert!(new_admissions[0].contains("ready=true"));
16240
16241 let old_admissions: Vec<_> = captured
16242 .lines()
16243 .filter(|line| {
16244 line.contains("module registered module_id=aft ")
16245 && line.contains("connection_id=30")
16246 })
16247 .collect();
16248 assert_eq!(
16249 old_admissions.len(),
16250 1,
16251 "unexpected incumbent admission: {captured}"
16252 );
16253
16254 let old_ends: Vec<_> = captured
16255 .lines()
16256 .filter(|line| {
16257 line.contains("connection_id=30") && line.contains("module registration ended")
16258 })
16259 .collect();
16260 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
16261 assert!(old_ends[0].contains("module_id=aft"));
16262 assert!(old_ends[0].contains("connection_id=30"));
16263 assert!(old_ends[0].contains("reason=replaced"));
16264 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
16265 assert_eq!(
16266 captured
16267 .lines()
16268 .filter(|line| line.contains("module registration promoted"))
16269 .count(),
16270 1,
16271 "unexpected promotion logs: {captured}"
16272 );
16273 }
16274
16275 #[tokio::test]
16281 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
16282 let mut swap = swap_with_incumbent().await;
16283 let handler = swap.handler.clone();
16284
16285 let cotenant = ConnectionId::new(31);
16287 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
16288 let (cotenant_task, cotenant_bind) = relay_route_open(
16289 &handler,
16290 cotenant,
16291 &cotenant_ctx.egress,
16292 &mut swap.incumbent_rx,
16293 100,
16294 "aft",
16295 "swap-cotenant",
16296 )
16297 .await;
16298 handler
16299 .handle_control_frame(
16300 &swap.incumbent_ctx,
16301 route_bind_ack(cotenant_bind.header.corr),
16302 )
16303 .await
16304 .unwrap();
16305 assert!(cotenant_task.await.unwrap().is_empty());
16306 let (cotenant_channel, cotenant_epoch) =
16307 published_route(&cotenant_rx.recv().await.unwrap());
16308
16309 let caller = ConnectionId::new(32);
16311 let (caller_ctx, mut caller_rx) = route_ctx(caller);
16312 let (caller_task, caller_bind) = relay_route_open(
16313 &handler,
16314 caller,
16315 &caller_ctx.egress,
16316 &mut swap.incumbent_rx,
16317 101,
16318 "aft",
16319 "swap-caller",
16320 )
16321 .await;
16322 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
16323
16324 register_candidate(&swap, None);
16325 cutover(&swap);
16326
16327 let ack = handler
16329 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
16330 .await;
16331 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
16332 if module_loop_error.is_some() {
16333 handler.cleanup_connection(INCUMBENT).unwrap();
16336 }
16337
16338 assert!(
16340 cotenant_rx.try_recv().is_err(),
16341 "the co-tenant route on the incumbent was torn down by one late ack: \
16342 {module_loop_error:?}"
16343 );
16344 assert!(matches!(
16345 swap.forwarding
16346 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
16347 .unwrap(),
16348 DataRoute::Client(DataRouteState::Bound(_))
16349 ));
16350 assert_eq!(module_loop_error, None);
16351 assert!(swap
16352 .registry
16353 .get_module_by_connection(INCUMBENT)
16354 .unwrap()
16355 .is_some());
16356
16357 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
16359 .await
16360 .expect("the incumbent is told to drop the abandoned binding")
16361 .unwrap()
16362 .frame;
16363 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16364 assert_eq!(goodbye.header.channel, abandoned_channel);
16365 assert_eq!(goodbye.header.epoch, abandoned_epoch);
16366 assert!(swap.incumbent_rx.try_recv().is_err());
16367
16368 let response = caller_task.await.unwrap();
16370 assert_eq!(response.len(), 1);
16371 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
16372 assert!(caller_rx.try_recv().is_err());
16373
16374 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
16378 let counters = handler.counters().snapshot();
16379 assert_eq!(
16380 keyed_total(&counters, "route_open_accepted_by_principal"),
16381 1
16382 );
16383 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
16384 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
16385 }
16386
16387 #[tokio::test]
16391 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
16392 let mut swap = swap_with_incumbent().await;
16393 register_candidate(&swap, None);
16394 let incumbent = cutover(&swap);
16395 swap.forwarding
16396 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
16397 .unwrap()
16398 .expect("the incumbent is still registered");
16399
16400 let client = ConnectionId::new(33);
16401 let (client_ctx, mut client_rx) = route_ctx(client);
16402 let route_handler = swap.handler.clone();
16403 let open_ctx = RouteCtx {
16404 connection_id: client,
16405 egress: client_ctx.egress.clone(),
16406 };
16407 let mut route_task = tokio::spawn(async move {
16408 route_handler
16409 .handle_control_frame(
16410 &open_ctx,
16411 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
16412 )
16413 .await
16414 .unwrap()
16415 });
16416 let bind = tokio::select! {
16417 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
16418 response = &mut route_task => {
16419 let response = response.unwrap();
16420 panic!(
16421 "post-cutover route.open was refused instead of relayed to the candidate: {}",
16422 parse_error(&response[0])["code"]
16423 );
16424 }
16425 };
16426 swap.handler
16427 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
16428 .await
16429 .unwrap();
16430 assert!(route_task.await.unwrap().is_empty());
16431 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
16432 match swap
16433 .forwarding
16434 .lookup_data_route(client, channel, epoch)
16435 .unwrap()
16436 {
16437 DataRoute::Client(DataRouteState::Bound(route)) => {
16438 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
16439 }
16440 other => panic!("expected a bound route on the candidate, got {other:?}"),
16441 }
16442 assert!(swap.incumbent_rx.try_recv().is_err());
16443 }
16444
16445 #[tokio::test]
16450 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
16451 let swap = swap_with_incumbent().await;
16452 register_candidate(&swap, Some(false));
16453 let update = Frame::build(
16454 FrameType::Request,
16455 control_flags(),
16456 0,
16457 0,
16458 55,
16459 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16460 provides: manifest("aft", PROTOCOL_VERSION).provides,
16461 capabilities: None,
16462 ready: Some(true),
16463 })
16464 .unwrap(),
16465 )
16466 .unwrap();
16467
16468 let replies = swap
16469 .handler
16470 .handle_control_frame(&swap.candidate_ctx, update)
16471 .await
16472 .unwrap();
16473
16474 assert_eq!(replies.len(), 1);
16475 assert_eq!(
16476 replies[0].header.ty,
16477 FrameType::Response,
16478 "candidate catalog.update was refused: {:?}",
16479 serde_json::from_slice::<Value>(&replies[0].body).ok()
16480 );
16481 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
16482 assert_eq!(
16483 swap.registry
16484 .get_module("aft")
16485 .unwrap()
16486 .unwrap()
16487 .connection_id,
16488 INCUMBENT
16489 );
16490 }
16491 }
16492
16493 mod swap_admission {
16497 use super::*;
16498
16499 const INCUMBENT_NONCE: &str = "incumbent-nonce";
16500 const CANDIDATE_NONCE: &str = "candidate-nonce";
16501
16502 fn handler_with_incumbent(
16503 module_id: &str,
16504 reserved: bool,
16505 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
16506 let registry = Arc::new(Registry::default());
16507 let supervisor = SupervisorHandle::new();
16508 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
16509 if reserved {
16510 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
16511 }
16512 let handler =
16513 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
16514 let incumbent = handler
16515 .handle_control(
16516 ConnectionId::new(1),
16517 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
16518 )
16519 .unwrap();
16520 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
16521 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
16522 (registry, supervisor, handler)
16523 }
16524
16525 #[test]
16531 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
16532 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
16533
16534 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
16535 let replies = handler
16536 .handle_control(
16537 ConnectionId::new(connection),
16538 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
16539 )
16540 .unwrap();
16541 assert_eq!(replies[0].header.ty, FrameType::Error);
16542 assert_eq!(
16543 parse_error(&replies[0])["code"],
16544 "swap_token_invalid",
16545 "nonce {nonce:?}"
16546 );
16547 }
16548 assert!(registry.get_candidate("aft").unwrap().is_none());
16549 assert_eq!(
16550 registry.get_module("aft").unwrap().unwrap().connection_id,
16551 ConnectionId::new(1)
16552 );
16553
16554 let admitted = handler
16557 .handle_control(
16558 ConnectionId::new(4),
16559 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
16560 )
16561 .unwrap();
16562 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
16563 assert_eq!(
16564 registry
16565 .get_candidate("aft")
16566 .unwrap()
16567 .unwrap()
16568 .connection_id,
16569 ConnectionId::new(4)
16570 );
16571 assert_eq!(
16572 registry.get_module("aft").unwrap().unwrap().connection_id,
16573 ConnectionId::new(1),
16574 "the candidate must not take the active slot"
16575 );
16576 let replayed = handler
16577 .handle_control(
16578 ConnectionId::new(5),
16579 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
16580 )
16581 .unwrap();
16582 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
16583
16584 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
16588 let squatter = handler
16589 .handle_control(
16590 ConnectionId::new(6),
16591 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
16592 )
16593 .unwrap();
16594 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
16595 assert!(
16596 registry.get_module("aft").unwrap().is_none(),
16597 "a squatter took the active slot of an id being swapped"
16598 );
16599 }
16600
16601 #[test]
16606 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
16607 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
16608
16609 let replies = handler
16610 .handle_control(
16611 ConnectionId::new(2),
16612 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16613 )
16614 .unwrap();
16615
16616 assert_eq!(
16617 replies[0].header.ty,
16618 FrameType::HelloAck,
16619 "reserved candidate refused: {:?}",
16620 serde_json::from_slice::<Value>(&replies[0].body).ok()
16621 );
16622 assert_eq!(
16623 registry
16624 .get_candidate("vault")
16625 .unwrap()
16626 .unwrap()
16627 .connection_id,
16628 ConnectionId::new(2)
16629 );
16630 }
16631
16632 #[test]
16635 fn without_an_open_swap_the_ordinary_gates_decide() {
16636 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
16637 supervisor.close_swap("vault");
16638
16639 let candidate = handler
16640 .handle_control(
16641 ConnectionId::new(2),
16642 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
16643 )
16644 .unwrap();
16645 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
16646 let duplicate = handler
16647 .handle_control(
16648 ConnectionId::new(3),
16649 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
16650 )
16651 .unwrap();
16652 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
16653 assert!(registry.get_candidate("vault").unwrap().is_none());
16654 }
16655 }
16656
16657 mod scopes {
16661 mod idle_tests {
16662 use super::*;
16663 include!("scope_idle_tests.rs");
16664 }
16665 mod route_open_timings {
16666 use super::*;
16667 include!("route_open_timing_tests.rs");
16668 }
16669 mod route_open_refusal_log {
16670 use super::*;
16671 include!("route_open_refusal_log_tests.rs");
16672 }
16673 mod activity {
16674 include!("activity_scope_tests.rs");
16675 }
16676 mod delta {
16677 include!("scope_handler_delta_tests.rs");
16678 }
16679 mod call_modes {
16680 include!("scope_call_mode_tests.rs");
16681 }
16682 #[tokio::test]
16683 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
16684 let rig = rig().await;
16685 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
16686 "s".into(),
16687 1,
16688 "tool".into(),
16689 ))
16690 .unwrap();
16691 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
16692 let (direct, _rx) = wide_ctx(999);
16693 for (ctx, expected) in [
16694 (&rig.owner, FrameType::Response),
16695 (&direct, FrameType::Error),
16696 ] {
16697 let frame =
16698 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
16699 .unwrap();
16700 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
16701 assert_eq!(replies[0].header.ty, expected);
16702 if expected == FrameType::Response {
16703 let reply: subc_control::ScopeServedByReply =
16704 serde_json::from_slice(&replies[0].body).unwrap();
16705 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
16706 } else {
16707 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
16708 }
16709 }
16710 let reply = call(
16711 &rig.handler,
16712 &rig.owner,
16713 &ModuleControlRequestFromModule::ScopeDescribe {
16714 owner: Principal::Reserved {
16715 module_id: OWNER.into(),
16716 },
16717 scope_ref: "s".into(),
16718 },
16719 )
16720 .await;
16721 assert!(matches!(
16722 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
16723 ModuleControlResponseToModule::ScopeDescribe { .. }
16724 ));
16725 }
16726
16727 use subc_protocol::scope::{
16728 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
16729 ScopeStatus,
16730 };
16731
16732 use super::*;
16733
16734 const OWNER: &str = "prefrontal-core";
16735
16736 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
16737 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
16738 }
16739
16740 async fn call(
16741 handler: &ControlHandler,
16742 ctx: &RouteCtx,
16743 request: &ModuleControlRequestFromModule,
16744 ) -> Frame {
16745 let body = serde_json::to_vec(request).unwrap();
16746 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
16747 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
16748 assert_eq!(replies.len(), 1, "{replies:?}");
16749 replies.pop().unwrap()
16750 }
16751
16752 async fn sync(
16753 handler: &ControlHandler,
16754 ctx: &RouteCtx,
16755 generation: u64,
16756 scopes: Vec<ScopeRecord>,
16757 ) -> Result<ModuleControlResponseToModule, String> {
16758 let reply = call(
16759 handler,
16760 ctx,
16761 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
16762 )
16763 .await;
16764 match reply.header.ty {
16765 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
16766 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
16767 }
16768 }
16769
16770 async fn describe(
16771 handler: &ControlHandler,
16772 ctx: &RouteCtx,
16773 owner: &str,
16774 scope_ref: &str,
16775 ) -> ModuleControlResponseToModule {
16776 let reply = call(
16777 handler,
16778 ctx,
16779 &ModuleControlRequestFromModule::ScopeDescribe {
16780 owner: Principal::Reserved {
16781 module_id: owner.to_string(),
16782 },
16783 scope_ref: scope_ref.to_string(),
16784 },
16785 )
16786 .await;
16787 assert_eq!(
16788 reply.header.ty,
16789 FrameType::Response,
16790 "{:?}",
16791 parse_error(&reply)
16792 );
16793 serde_json::from_slice(&reply.body).unwrap()
16794 }
16795
16796 #[tokio::test]
16797 async fn scope_describe_never_echoes_bind_proof_or_hash() {
16798 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
16799 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
16800 let mut rig = rig().await;
16801 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
16802 .await;
16803 let _bound = rig
16804 .bound(
16805 Some(AFT),
16806 PLEXUS,
16807 Some(
16808 rig.selector("s", Some(1))
16809 .with_bind_proof(Some(PROOF.into())),
16810 ),
16811 )
16812 .await;
16813 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
16814 let output = serde_json::to_string(&reply).unwrap();
16815 assert!(output.contains("\"status\":\"live\""));
16816 assert!(!output.contains(PROOF));
16817 assert!(!output.contains(HASH));
16818 }
16819
16820 async fn module(
16822 handler: &ControlHandler,
16823 connection: u64,
16824 module_id: &str,
16825 nonce: Option<&str>,
16826 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
16827 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
16828 hello_via_sink(
16829 handler,
16830 &ctx,
16831 &mut rx,
16832 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
16833 )
16834 .await;
16835 (ctx, rx)
16836 }
16837
16838 #[tokio::test]
16841 async fn a_client_connection_cannot_sync_or_describe() {
16842 let handler = ControlHandler::new(Arc::new(Registry::default()));
16843 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
16844 for request in [
16845 ModuleControlRequestFromModule::ScopeSync {
16846 generation: 1,
16847 scopes: vec![head("s", 1)],
16848 },
16849 ModuleControlRequestFromModule::ScopeDescribe {
16850 owner: Principal::Direct,
16851 scope_ref: "s".to_string(),
16852 },
16853 ] {
16854 let reply = call(&handler, &ctx, &request).await;
16855 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
16856 }
16857 assert!(
16858 !handler
16859 .scopes
16860 .read()
16861 .unwrap()
16862 .describe(
16863 &Principal::Reserved {
16864 module_id: OWNER.to_string()
16865 },
16866 "s"
16867 )
16868 .owner_synced
16869 );
16870 }
16871
16872 #[tokio::test]
16875 async fn a_module_without_a_supervised_launch_cannot_sync() {
16876 let handler = ControlHandler::new(Arc::new(Registry::default()));
16877 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
16878 assert_eq!(
16879 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
16880 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16881 );
16882 }
16883
16884 #[tokio::test]
16885 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
16886 let supervisor = SupervisorHandle::new();
16887 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16888 let handler = ControlHandler::new(Arc::new(Registry::default()))
16889 .with_supervisor(supervisor.clone());
16890 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16891 sync(&handler, &incumbent, 1, vec![head("s", 1)])
16892 .await
16893 .expect("the current launch syncs");
16894
16895 supervisor.open_swap(OWNER, "n2".to_string());
16898 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
16899 assert_eq!(
16900 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16901 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16902 );
16903 sync(&handler, &incumbent, 2, vec![head("s", 1)])
16904 .await
16905 .expect("the serving owner syncs during the swap");
16906
16907 supervisor.close_swap(OWNER);
16910 assert_eq!(
16911 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
16912 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16913 );
16914 sync(&handler, &incumbent, 3, vec![head("s", 1)])
16915 .await
16916 .expect("the serving owner syncs after the rollback");
16917 handler.cleanup_connection(candidate.connection_id).unwrap();
16918
16919 supervisor.open_swap(OWNER, "n3".to_string());
16924 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
16925 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
16926 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
16927 .await
16928 .expect("the promoted launch takes authority");
16929 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
16930 panic!("unexpected reply {reply:?}");
16931 };
16932 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
16933 assert_eq!(
16934 sync(&handler, &incumbent, 4, Vec::new()).await,
16935 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
16936 );
16937 }
16938
16939 #[tokio::test]
16942 async fn closing_the_authority_connection_frees_sync_authority() {
16943 let supervisor = SupervisorHandle::new();
16944 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16945 let handler = ControlHandler::new(Arc::new(Registry::default()))
16946 .with_supervisor(supervisor.clone());
16947 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
16948 sync(&handler, &first, 10, vec![head("s", 1)])
16949 .await
16950 .unwrap();
16951 handler.cleanup_connection(first.connection_id).unwrap();
16952
16953 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
16954 sync(&handler, &second, 1, vec![head("s", 1)])
16955 .await
16956 .expect("the next connection takes the released authority");
16957 }
16958
16959 #[tokio::test]
16960 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
16961 let supervisor = SupervisorHandle::new();
16962 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
16963 let handler =
16964 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
16965 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
16966 sync(&handler, &first, 10, vec![head("s", 1)])
16967 .await
16968 .unwrap();
16969 handler
16970 .handle_control_frame(
16971 &first,
16972 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
16973 )
16974 .await
16975 .unwrap();
16976 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
16977 sync(&handler, &second, 1, vec![head("s", 1)])
16978 .await
16979 .expect("GOODBYE releases authority even if the old socket remains open");
16980 }
16981
16982 #[tokio::test]
16983 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
16984 let registry = Arc::new(Registry::default());
16985 let supervisor_handle = SupervisorHandle::new();
16986 let supervisor =
16987 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
16988 .with_handle(supervisor_handle.clone())
16989 .with_daemon_incarnation("incarnation-7".to_string());
16990 supervisor
16993 .supervise_configured(
16994 ModuleSpec {
16995 module_id: OWNER.to_string(),
16996 program: PathBuf::from("/nonexistent/prefrontal-core"),
16997 args: Vec::new(),
16998 env: Vec::new(),
16999 reserved: false,
17000 reserved_prefixes: Vec::new(),
17001 protocol: ModuleProtocol::Subc,
17002 overlap: Default::default(),
17003 },
17004 false,
17005 )
17006 .unwrap();
17007 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
17008 let handler =
17009 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
17010 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
17011
17012 let ModuleControlResponseToModule::ScopeDescribe {
17014 status,
17015 daemon_incarnation,
17016 owner_synced,
17017 owner_configured,
17018 scope,
17019 ..
17020 } = describe(&handler, &reader, OWNER, "s").await
17021 else {
17022 panic!("not a describe reply");
17023 };
17024 assert_eq!(status, ScopeStatus::NotLive);
17025 assert_eq!(daemon_incarnation, "incarnation-7");
17026 assert!(!owner_synced);
17027 assert!(owner_configured);
17028 assert!(scope.is_none());
17029
17030 let ModuleControlResponseToModule::ScopeDescribe {
17033 status,
17034 owner_configured,
17035 ..
17036 } = describe(&handler, &reader, "ghost", "s").await
17037 else {
17038 panic!("not a describe reply");
17039 };
17040 assert_eq!(status, ScopeStatus::NotLive);
17041 assert!(!owner_configured);
17042
17043 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
17045 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
17046 let ModuleControlResponseToModule::ScopeDescribe {
17047 status,
17048 scope_epoch,
17049 owner_synced,
17050 scope,
17051 ..
17052 } = describe(&handler, &reader, OWNER, "s").await
17053 else {
17054 panic!("not a describe reply");
17055 };
17056 assert_eq!(status, ScopeStatus::Live);
17057 assert_eq!(scope_epoch, Some(4));
17058 assert!(owner_synced);
17059 let stamp = scope.expect("a live scope carries its stamp");
17060 assert!(
17061 stamp.owner_authorized,
17062 "prefrontal-core is the default authority"
17063 );
17064 assert_eq!(stamp.kind, ScopeKind::Head);
17065 }
17066
17067 #[tokio::test]
17068 async fn scope_authority_owners_decides_owner_authorized() {
17069 let supervisor = SupervisorHandle::new();
17070 supervisor.set_spawn_nonce("broca", "b1".to_string());
17071 let handler = ControlHandler::new(Arc::new(Registry::default()))
17072 .with_supervisor(supervisor)
17073 .with_scope_authority_owners(vec!["broca".to_string()]);
17074 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
17075 let mut gated = head("s", 1);
17076 gated.attributes.agent_id = Some("agent".to_string());
17077 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
17078 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
17079 describe(&handler, &broca, "broca", "s").await
17080 else {
17081 panic!("not a describe reply");
17082 };
17083 assert!(scope.unwrap().owner_authorized);
17084 }
17085
17086 #[tokio::test]
17090 async fn scope_ops_and_the_scopes_capability_are_advertised() {
17091 let handler = ControlHandler::new(Arc::new(Registry::default()));
17092 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
17093 let ack = hello_via_sink(
17094 &handler,
17095 &ctx,
17096 &mut rx,
17097 hello_frame("m", PROTOCOL_VERSION, 1),
17098 )
17099 .await;
17100 let ack = parse_ack(&ack);
17101 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
17102 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
17103 }
17104 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
17105
17106 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
17107 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
17108 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
17109 let reply = handler
17110 .handle_control_frame(&client, frame)
17111 .await
17112 .unwrap()
17113 .pop()
17114 .unwrap();
17115 let ClientControlResponse::ServerDescribe { capabilities, .. } =
17116 serde_json::from_slice(&reply.body).unwrap()
17117 else {
17118 panic!("not a server.describe reply");
17119 };
17120 assert!(
17121 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
17122 "{capabilities:?}"
17123 );
17124 }
17125
17126 const PLEXUS: &str = "plexus";
17129 const OTHER: &str = "other";
17130 const AFT: &str = "aft";
17131 const BROCA: &str = "broca";
17132 const MAGIC: &str = "magic-context";
17133
17134 fn nonce(module_id: &str) -> String {
17135 format!("nonce-{module_id}")
17136 }
17137
17138 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17139 let (tx, rx) = mpsc::channel(64);
17140 (
17141 RouteCtx {
17142 connection_id: ConnectionId::new(connection),
17143 egress: FrameSink::new(tx),
17144 },
17145 rx,
17146 )
17147 }
17148
17149 struct Rig {
17153 handler: ControlHandler,
17154 forwarding: Arc<ForwardingTable>,
17155 owner: RouteCtx,
17156 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17157 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
17158 generation: u64,
17159 next_connection: u64,
17160 _supervisor: Supervisor,
17161 }
17162
17163 async fn rig() -> Rig {
17164 rig_with_flow_support(true).await
17165 }
17166
17167 async fn rig_with_flow_support(flow_support: bool) -> Rig {
17168 rig_with_scope_capabilities(flow_support, false).await
17169 }
17170
17171 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
17172 let mut provides = Vec::new();
17173 if flow_support {
17174 provides.push("flow-scopes/v1");
17175 }
17176 if run_support {
17177 provides.push("agent-run-scopes/v1");
17178 }
17179 rig_with_target_capabilities(&provides).await
17180 }
17181
17182 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
17183 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
17184 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17185 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17188 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
17189 Frame::build(
17190 FrameType::Hello,
17191 control_flags(),
17192 0,
17193 0,
17194 connection,
17195 serde_json::to_vec(&body).unwrap(),
17196 )
17197 .unwrap()
17198 }
17199
17200 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
17201 let registry = Arc::new(Registry::default());
17202 let forwarding = Arc::new(ForwardingTable::default());
17203 let supervisor_handle = SupervisorHandle::new();
17204 let supervisor =
17205 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
17206 .with_handle(supervisor_handle.clone());
17207 supervisor
17208 .supervise_configured(
17209 ModuleSpec {
17210 module_id: OWNER.to_string(),
17211 program: PathBuf::from("/nonexistent/prefrontal-core"),
17212 args: Vec::new(),
17213 env: Vec::new(),
17214 reserved: false,
17215 reserved_prefixes: Vec::new(),
17216 protocol: ModuleProtocol::Subc,
17217 overlap: Default::default(),
17218 },
17219 false,
17220 )
17221 .unwrap();
17222 for module_id in [OWNER, AFT, BROCA, MAGIC] {
17223 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
17224 }
17225 let handler =
17226 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17227 .with_supervisor(supervisor_handle);
17228 let (owner, mut owner_rx) = wide_ctx(1);
17229 hello_via_sink(
17230 &handler,
17231 &owner,
17232 &mut owner_rx,
17233 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
17234 )
17235 .await;
17236 let mut modules = BTreeMap::new();
17237 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
17238 let (ctx, mut rx) = wide_ctx(connection);
17239 let hello = hello_with_capabilities(module_id, connection, provides);
17240 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
17241 modules.insert(module_id.to_string(), (ctx, rx));
17242 }
17243 Rig {
17244 handler,
17245 forwarding,
17246 owner,
17247 _owner_rx: owner_rx,
17248 modules,
17249 generation: 0,
17250 next_connection: 100,
17251 _supervisor: supervisor,
17252 }
17253 }
17254
17255 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
17256 ScopeCarrier::new(Principal::Reserved {
17257 module_id: module_id.to_string(),
17258 })
17259 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
17260 }
17261
17262 fn session(scope_epoch: u64) -> ScopeRecord {
17265 let mut record = head("s", scope_epoch);
17266 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
17267 record.attributes.agent_id = Some("agent-1".to_string());
17268 record.attributes.delegates = true;
17269 record
17270 }
17271
17272 impl Rig {
17273 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
17274 self.generation += 1;
17275 sync(&self.handler, &self.owner, self.generation, scopes)
17276 .await
17277 .expect("the owner's sync is accepted");
17278 }
17279
17280 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17281 ScopeSelector::new(
17282 Principal::Reserved {
17283 module_id: OWNER.to_string(),
17284 },
17285 scope_ref.to_string(),
17286 scope_epoch,
17287 )
17288 }
17289
17290 fn open_frame(
17291 &mut self,
17292 opener: Option<&str>,
17293 target: &str,
17294 scope: Option<ScopeSelector>,
17295 ) -> (
17296 RouteCtx,
17297 mpsc::Receiver<crate::router::OutboundFrame>,
17298 Frame,
17299 ) {
17300 self.next_connection += 1;
17301 let (ctx, rx) = wide_ctx(self.next_connection);
17302 let root = unique_project_root("scoped-open");
17303 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
17304 target: RouteTarget::ToolProvider {
17305 module_id: target.to_string(),
17306 },
17307 identity: BindIdentity::new(
17308 root.path().to_path_buf(),
17309 "unit".to_string(),
17310 "session".to_string(),
17311 ),
17312 consumer_identity: opener.map(|module_id| ConsumerIdentity {
17313 module_id: module_id.to_string(),
17314 launch_nonce: nonce(module_id),
17315 }),
17316 consumer_capabilities: None,
17317 role_versions: None,
17318 admission_facts: None,
17319 scope,
17320 })
17321 .unwrap();
17322 let frame = Frame::build(
17323 FrameType::Request,
17324 control_flags(),
17325 0,
17326 0,
17327 self.next_connection,
17328 body,
17329 )
17330 .unwrap();
17331 (ctx, rx, frame)
17332 }
17333
17334 async fn refused(
17336 &mut self,
17337 opener: Option<&str>,
17338 target: &str,
17339 scope: Option<ScopeSelector>,
17340 ) -> String {
17341 self.refusal_body(opener, target, scope).await["code"]
17342 .as_str()
17343 .unwrap()
17344 .to_string()
17345 }
17346
17347 async fn refusal_body(
17348 &mut self,
17349 opener: Option<&str>,
17350 target: &str,
17351 scope: Option<ScopeSelector>,
17352 ) -> Value {
17353 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
17354 let replies = tokio::time::timeout(
17355 Duration::from_secs(2),
17356 self.handler.handle_control_frame(&ctx, frame),
17357 )
17358 .await
17359 .expect("the open must be refused before waiting for a bind ack")
17360 .unwrap();
17361 assert_eq!(replies.len(), 1, "{replies:?}");
17362 assert_eq!(replies[0].header.ty, FrameType::Error);
17363 let (_, module_rx) = self.modules.get_mut(target).unwrap();
17364 assert!(
17365 module_rx.try_recv().is_err(),
17366 "a refused open relays nothing"
17367 );
17368 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
17369 parse_error(&replies[0])
17370 }
17371
17372 async fn relayed(
17374 &mut self,
17375 opener: Option<&str>,
17376 target: &str,
17377 scope: Option<ScopeSelector>,
17378 ) -> Relayed {
17379 let (ctx, rx, frame) = self.open_frame(opener, target, scope);
17380 let handler = self.handler.clone();
17381 let task_ctx = ctx.clone();
17382 let task = tokio::spawn(async move {
17383 handler
17384 .handle_control_frame(&task_ctx, frame)
17385 .await
17386 .unwrap()
17387 });
17388 let (_, module_rx) = self.modules.get_mut(target).unwrap();
17389 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
17390 .await
17391 .expect("the target receives the relayed route.bind")
17392 .unwrap()
17393 .frame;
17394 Relayed {
17395 target: target.to_string(),
17396 client: ctx,
17397 client_rx: rx,
17398 task,
17399 bind,
17400 }
17401 }
17402
17403 async fn ack(&self, relayed: &Relayed) {
17404 let (module, _) = &self.modules[&relayed.target];
17405 self.handler
17406 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
17407 .await
17408 .unwrap();
17409 }
17410
17411 async fn bound(
17413 &mut self,
17414 opener: Option<&str>,
17415 target: &str,
17416 scope: Option<ScopeSelector>,
17417 ) -> Bound {
17418 let relayed = self.relayed(opener, target, scope).await;
17419 self.ack(&relayed).await;
17420 let Relayed {
17421 target,
17422 client,
17423 mut client_rx,
17424 task,
17425 bind,
17426 } = relayed;
17427 assert!(
17428 task.await.unwrap().is_empty(),
17429 "the open is answered by commit"
17430 );
17431 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
17432 Bound {
17433 target,
17434 client,
17435 client_rx,
17436 channel,
17437 epoch,
17438 bind,
17439 }
17440 }
17441
17442 fn live(&self, route: &Bound) -> bool {
17443 matches!(
17444 self.forwarding
17445 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
17446 .unwrap(),
17447 DataRoute::Client(DataRouteState::Bound(_))
17448 )
17449 }
17450 }
17451
17452 struct Relayed {
17453 target: String,
17454 client: RouteCtx,
17455 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17456 task: tokio::task::JoinHandle<Vec<Frame>>,
17457 bind: Frame,
17458 }
17459
17460 struct Bound {
17461 target: String,
17462 client: RouteCtx,
17463 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17464 channel: u16,
17465 epoch: u32,
17466 bind: Frame,
17467 }
17468
17469 impl Bound {
17470 fn closed_reason(&mut self) -> RouteCloseReason {
17473 let mut reason = None;
17474 let mut goodbye = false;
17475 while let Ok(outbound) = self.client_rx.try_recv() {
17476 let frame = outbound.frame;
17477 match frame.header.ty {
17478 FrameType::Goodbye => {
17479 assert_eq!(
17480 (frame.header.channel, frame.header.epoch),
17481 (self.channel, self.epoch)
17482 );
17483 goodbye = true;
17484 }
17485 FrameType::Push => {
17486 let ClientControlPush::RouteClosed {
17487 reason: r,
17488 module_id,
17489 ..
17490 } = serde_json::from_slice(&frame.body).unwrap()
17491 else {
17492 panic!("unexpected push");
17493 };
17494 assert_eq!(module_id, self.target);
17495 reason = Some(r);
17496 }
17497 other => panic!("unexpected frame {other:?}"),
17498 }
17499 }
17500 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
17501 reason.expect("the client is told why the route closed")
17502 }
17503
17504 fn untouched(&mut self) -> bool {
17505 self.client_rx.try_recv().is_err()
17506 }
17507
17508 fn stamp(&self) -> Option<ScopeStamp> {
17509 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
17510 ModuleControlRequest::RouteBind { scope, .. } => scope,
17511 other => panic!("expected a route.bind, got {other:?}"),
17512 }
17513 }
17514 }
17515
17516 #[tokio::test]
17517 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
17518 ) {
17519 let mut rig = rig().await;
17520 let mut record = session(1);
17521 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
17522 record.child_owners = vec![Principal::Reserved {
17523 module_id: MAGIC.to_string(),
17524 }];
17525 rig.sync(vec![record]).await;
17526 let scope = || Some(rig_selector("s", Some(1)));
17527
17528 rig.bound(Some(OWNER), PLEXUS, scope()).await;
17531 rig.bound(Some(AFT), OTHER, scope()).await;
17532 rig.bound(Some(BROCA), PLEXUS, scope()).await;
17533
17534 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
17538 assert_eq!(
17539 rig.refused(opener, target, scope()).await,
17540 error_codes::SCOPE_NOT_CARRIER,
17541 "{opener:?} -> {target}"
17542 );
17543 }
17544 }
17545
17546 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
17547 ScopeSelector::new(
17548 Principal::Reserved {
17549 module_id: OWNER.to_string(),
17550 },
17551 scope_ref.to_string(),
17552 scope_epoch,
17553 )
17554 }
17555
17556 #[tokio::test]
17557 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
17558 let mut rig = rig().await;
17559 rig.sync(vec![session(1)]).await;
17560 for opener in [OWNER, AFT] {
17561 assert_eq!(
17562 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
17563 .await,
17564 error_codes::SCOPE_EPOCH_REQUIRED,
17565 "{opener}"
17566 );
17567 }
17568 }
17569
17570 #[tokio::test]
17571 async fn admission_separates_not_synced_not_live_and_ended() {
17572 let mut rig = rig().await;
17573 let code = rig
17575 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17576 .await;
17577 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
17578 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
17579
17580 let ghost = ScopeSelector::new(
17582 Principal::Reserved {
17583 module_id: "ghost".to_string(),
17584 },
17585 "s".to_string(),
17586 Some(1),
17587 );
17588 assert_eq!(
17589 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
17590 error_codes::SCOPE_NOT_LIVE
17591 );
17592
17593 rig.sync(vec![session(2)]).await;
17594 assert_eq!(
17595 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
17596 .await,
17597 error_codes::SCOPE_NOT_LIVE
17598 );
17599 for epoch in [1, 3] {
17600 assert_eq!(
17601 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
17602 .await,
17603 error_codes::SCOPE_ENDED,
17604 "epoch {epoch}"
17605 );
17606 }
17607 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
17609 .await;
17610 }
17611
17612 #[tokio::test]
17613 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
17614 let mut rig = rig().await;
17615 rig.sync(vec![session(1)]).await;
17616 let route = rig
17617 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17618 .await;
17619 let stamp = route.stamp().expect("a scoped bind carries the stamp");
17620 assert_eq!(stamp.scope_ref, "s");
17621 assert_eq!(stamp.scope_epoch, 1);
17622 assert_eq!(stamp.kind, ScopeKind::Head);
17623 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
17624 assert!(stamp.attributes.delegates);
17625 assert!(stamp.owner_authorized);
17626 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17627 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
17628
17629 let (broca, mut broca_rx) = wide_ctx(50);
17632 hello_via_sink(
17633 &rig.handler,
17634 &broca,
17635 &mut broca_rx,
17636 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
17637 )
17638 .await;
17639 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
17640 .await
17641 .unwrap();
17642 let own = ScopeSelector::new(
17643 Principal::Reserved {
17644 module_id: BROCA.to_string(),
17645 },
17646 "b".to_string(),
17647 Some(1),
17648 );
17649 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
17650 assert!(!route.stamp().unwrap().owner_authorized);
17651 }
17652
17653 #[tokio::test]
17654 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
17655 let mut rig = rig().await;
17656 let mut record = head("s", 1);
17657 record.carriers = vec![carrier(AFT, None)];
17658 let flow_id = "Flow:run-7/step_2!~";
17659 record.attributes.flow_id = Some(flow_id.to_string());
17660 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
17661 .await
17662 .unwrap();
17663 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17664 panic!("not a sync reply");
17665 };
17666 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
17667 let route = rig
17668 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17669 .await;
17670 assert!(rig.live(&route), "the stamped bind committed");
17671 let stamp = route.stamp().expect("a flow scope carries a stamp");
17672 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
17673 assert_eq!(stamp.attributes.agent_id, None);
17674 assert!(!stamp.attributes.delegates);
17675 assert!(stamp.owner_authorized);
17676 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17677 assert_eq!(unscoped.stamp(), None);
17678 }
17679
17680 #[tokio::test]
17681 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
17682 let mut rig = rig_with_flow_support(false).await;
17683 let mut record = session(1);
17684 record.attributes.flow_id = Some("flow:7".to_string());
17685 rig.sync(vec![record]).await;
17686 for opener in [OWNER, AFT] {
17687 let body = rig
17688 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17689 .await;
17690 assert_eq!(body["code"], "target_flow_unsupported");
17691 let message = body["message"].as_str().unwrap();
17692 for required in [PLEXUS, "flow-scopes/v1"] {
17693 assert!(message.contains(required), "{message}");
17694 }
17695 }
17696 }
17697
17698 #[tokio::test]
17699 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
17700 let mut rig = rig_with_flow_support(true).await;
17701 let mut record = session(1);
17702 record.attributes.flow_id = Some("flow:7".to_string());
17703 rig.sync(vec![record]).await;
17704 for opener in [OWNER, AFT] {
17705 let route = rig
17706 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
17707 .await;
17708 assert!(rig.live(&route));
17709 assert_eq!(
17710 route.stamp().unwrap().attributes.flow_id.as_deref(),
17711 Some("flow:7")
17712 );
17713 }
17714 }
17715
17716 #[tokio::test]
17717 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
17718 use std::future::Future;
17719
17720 let mut rig = rig_with_flow_support(true).await;
17721 let mut record = session(1);
17722 record.attributes.flow_id = Some("flow:7".to_string());
17723 rig.sync(vec![record]).await;
17724 let (client, mut client_rx, frame) =
17725 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
17726 for _ in 0..64 {
17729 client.egress.try_send(route_bind_ack(1)).unwrap();
17730 }
17731 let handler = rig.handler.clone();
17732 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17733 std::future::poll_fn(|cx| {
17734 assert!(open.as_mut().poll(cx).is_pending());
17735 std::task::Poll::Ready(())
17736 })
17737 .await;
17738 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17739
17740 let old_connection = rig.modules[PLEXUS].0.connection_id;
17741 rig.handler.cleanup_connection(old_connection).unwrap();
17742 let (replacement, mut replacement_rx) = wide_ctx(200);
17743 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
17744 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
17745 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
17746 let hello = Frame::build(
17747 FrameType::Hello,
17748 control_flags(),
17749 0,
17750 0,
17751 200,
17752 serde_json::to_vec(&body).unwrap(),
17753 )
17754 .unwrap();
17755 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
17756
17757 client_rx.try_recv().unwrap();
17758 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17759 .await
17760 .expect("the replacement is refused without waiting for a bind ack")
17761 .unwrap();
17762 assert_eq!(replies.len(), 1);
17763 let body = parse_error(&replies[0]);
17764 assert_eq!(body["code"], "target_flow_unsupported");
17765 for required in [PLEXUS, "flow-scopes/v1"] {
17766 assert!(body["message"].as_str().unwrap().contains(required));
17767 }
17768 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
17769 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17770 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17771 assert!(rig
17772 .handler
17773 .registry
17774 .get_module_by_connection(replacement.connection_id)
17775 .unwrap()
17776 .is_some());
17777 }
17778
17779 #[tokio::test]
17780 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
17781 use std::future::Future;
17782 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17783 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17784 const NEXT_HASH: &str =
17785 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17786 let mut rig = rig().await;
17787 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
17788 record.carriers.push(ScopeCarrier::new(Principal::Direct));
17789 rig.sync(vec![record.clone()]).await;
17790 let (client, mut rx, frame) = rig.open_frame(
17791 None,
17792 PLEXUS,
17793 Some(
17794 rig.selector("s", Some(1))
17795 .with_bind_proof(Some(PROOF.into())),
17796 ),
17797 );
17798 for _ in 0..64 {
17801 client.egress.try_send(route_bind_ack(1)).unwrap();
17802 }
17803 let handler = rig.handler.clone();
17804 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17805 std::future::poll_fn(|cx| {
17806 assert!(open.as_mut().poll(cx).is_pending());
17807 std::task::Poll::Ready(())
17808 })
17809 .await;
17810 record.bind_proof_sha256 = Some(NEXT_HASH.into());
17811 rig.sync(vec![record]).await;
17812 rx.try_recv().unwrap();
17813 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17814 .await
17815 .unwrap()
17816 .unwrap();
17817 assert_eq!(
17818 parse_error(&replies[0])["code"],
17819 error_codes::SCOPE_BIND_PROOF_MISMATCH
17820 );
17821 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
17822 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
17823 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17824 }
17825
17826 #[tokio::test]
17827 async fn scope_bind_proof_reserved_openers_ignore_proof() {
17828 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17829 let mut rig = rig().await;
17830 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17831 .await;
17832 for opener in [OWNER, AFT] {
17833 for proof in [None, Some("malformed+proof=")] {
17834 let route = rig
17835 .bound(
17836 Some(opener),
17837 PLEXUS,
17838 Some(
17839 rig.selector("s", Some(1))
17840 .with_bind_proof(proof.map(str::to_string)),
17841 ),
17842 )
17843 .await;
17844 assert!(rig.live(&route));
17845 }
17846 }
17847 }
17848
17849 #[tokio::test]
17850 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
17851 use std::future::Future;
17852 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17853 const NEXT_HASH: &str =
17854 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
17855 let mut rig = rig().await;
17856 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17857 .await;
17858 let (client, mut rx, frame) =
17859 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
17860 for _ in 0..64 {
17861 client.egress.try_send(route_bind_ack(1)).unwrap();
17862 }
17863 let handler = rig.handler.clone();
17864 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
17865 std::future::poll_fn(|cx| {
17866 assert!(open.as_mut().poll(cx).is_pending());
17867 std::task::Poll::Ready(())
17868 })
17869 .await;
17870 rig.sync(vec![
17871 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
17872 ])
17873 .await;
17874 rx.try_recv().unwrap();
17875 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
17876 let bind = std::future::poll_fn(|cx| {
17877 assert!(open.as_mut().poll(cx).is_pending());
17878 module_rx.poll_recv(cx)
17879 })
17880 .await
17881 .unwrap()
17882 .frame;
17883 handler
17884 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
17885 .await
17886 .unwrap();
17887 let replies = tokio::time::timeout(Duration::from_secs(2), open)
17888 .await
17889 .unwrap()
17890 .unwrap();
17891 assert_eq!(replies.len(), 1);
17892 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
17895 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
17896 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
17897 let route = rig
17898 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
17899 .await;
17900 assert!(rig.live(&route));
17901 }
17902
17903 #[tokio::test]
17904 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
17905 {
17906 let mut rig = rig_with_flow_support(false).await;
17907 rig.sync(vec![session(1)]).await;
17908 let route = rig
17909 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17910 .await;
17911 assert!(rig.live(&route));
17912 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
17913 let mut record = session(1);
17914 record.attributes.flow_id = Some("flow:7".to_string());
17915 rig.sync(vec![record]).await;
17916 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
17917 assert!(rig.live(&unscoped));
17918 assert_eq!(unscoped.stamp(), None);
17919 }
17920
17921 #[tokio::test]
17922 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
17923 let mut rig = rig().await;
17924 let mut record = session(1);
17925 record.attributes.flow_id = Some("flow:7".to_string());
17926 rig.sync(vec![record.clone()]).await;
17927 let before = rig.forwarding.published_scope_tag(OWNER, "s");
17928 let mut owner_route = rig
17929 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17930 .await;
17931 let mut carrier_route = rig
17932 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
17933 .await;
17934 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
17935 rig.sync(vec![record.clone()]).await;
17936 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
17937 assert!(rig.live(&owner_route) && owner_route.untouched());
17938 assert!(rig.live(&carrier_route) && carrier_route.untouched());
17939
17940 record.attributes.flow_id = Some("flow:8".to_string());
17941 rig.sync(vec![record]).await;
17942 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
17943 let before = before.unwrap();
17944 assert_eq!(after.scope_epoch, before.scope_epoch);
17945 assert!(after.version > before.version);
17946 for route in [&mut owner_route, &mut carrier_route] {
17947 assert!(!rig.live(route));
17948 assert_eq!(
17949 route.closed_reason(),
17950 RouteCloseReason::ScopeDelegationChanged
17951 );
17952 }
17953 assert!(rig.live(&unscoped) && unscoped.untouched());
17954 for target in [PLEXUS, OTHER] {
17957 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
17958 let goodbye = module_rx
17959 .try_recv()
17960 .expect("the provider sees the drain")
17961 .frame;
17962 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17963 assert!(module_rx.try_recv().is_err());
17964 }
17965 let rebound = rig
17966 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17967 .await;
17968 assert_eq!(
17969 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
17970 Some("flow:8")
17971 );
17972 }
17973
17974 #[tokio::test]
17979 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
17980 let mut rig = rig().await;
17981 rig.sync(vec![session(1)]).await;
17982 let mut cotenant = rig
17983 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
17984 .await;
17985
17986 let mut changed = session(1);
17987 changed.child_owners.push(Principal::Reserved {
17988 module_id: MAGIC.to_string(),
17989 });
17990 let mut ended = None;
17991 for (code, next) in [
17992 (error_codes::SCOPE_CHANGED, vec![changed]),
17993 (error_codes::SCOPE_ENDED, Vec::new()),
17994 ] {
17995 let relayed = rig
17996 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
17997 .await;
17998 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
17999 ended = Some(next.is_empty());
18000 rig.sync(next).await;
18001 rig.ack(&relayed).await;
18002 let replies = relayed.task.await.unwrap();
18003 assert_eq!(replies.len(), 1, "{replies:?}");
18004 assert_eq!(parse_error(&replies[0])["code"], code);
18005 assert_eq!(
18006 subc_protocol::error_codes::is_retryable_route_open(code),
18007 code == error_codes::SCOPE_CHANGED
18008 );
18009 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18010 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18012 let mut goodbyes = Vec::new();
18015 while let Ok(outbound) = plexus_rx.try_recv() {
18016 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
18017 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
18018 }
18019 assert!(
18020 goodbyes.contains(&(bind_channel, bind_epoch)),
18021 "{goodbyes:?}"
18022 );
18023 assert!(rig
18024 .handler
18025 .registry
18026 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
18027 .unwrap()
18028 .is_some());
18029 }
18030 assert_eq!(ended, Some(true));
18031 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
18034 }
18035
18036 #[tokio::test]
18039 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
18040 struct Case {
18041 name: &'static str,
18042 change: fn(&mut ScopeRecord),
18043 closed: [Option<RouteCloseReason>; 4],
18046 }
18047 use RouteCloseReason::*;
18048 let cases = [
18049 Case {
18050 name: "a carrier entry removed",
18051 change: |r| {
18052 r.carriers.retain(|c| {
18053 c.principal
18054 != Principal::Reserved {
18055 module_id: AFT.to_string(),
18056 }
18057 })
18058 },
18059 closed: [None, Some(ScopeCarrierRemoved), None, None],
18060 },
18061 Case {
18062 name: "a target removed from a carrier",
18063 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
18064 closed: [None, None, None, Some(ScopeCarrierRemoved)],
18065 },
18066 Case {
18067 name: "a bare carrier narrowed to targets",
18068 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
18069 closed: [None, Some(ScopeCarrierRemoved), None, None],
18070 },
18071 Case {
18072 name: "delegates turned off",
18073 change: |r| r.attributes.delegates = false,
18074 closed: [Some(ScopeDelegationChanged); 4],
18075 },
18076 Case {
18077 name: "agent_id changed",
18078 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
18079 closed: [Some(ScopeDelegationChanged); 4],
18080 },
18081 Case {
18082 name: "a carrier added, child owners changed, the record re-sent",
18083 change: |r| {
18084 r.carriers.push(carrier(MAGIC, None));
18085 r.child_owners.push(Principal::Reserved {
18086 module_id: MAGIC.to_string(),
18087 });
18088 },
18089 closed: [None; 4],
18090 },
18091 Case {
18092 name: "a target added",
18093 change: |r| {
18094 r.carriers[1]
18095 .targets
18096 .as_mut()
18097 .unwrap()
18098 .push("third".to_string())
18099 },
18100 closed: [None; 4],
18101 },
18102 Case {
18103 name: "delegates turned on",
18104 change: |r| r.attributes.delegates = true,
18105 closed: [None; 4],
18106 },
18107 ];
18108 for case in cases {
18109 let mut rig = rig().await;
18110 rig.sync(vec![session(1)]).await;
18111 let scope = || Some(rig_selector("s", Some(1)));
18112 let mut routes = [
18113 rig.bound(Some(OWNER), PLEXUS, scope()).await,
18114 rig.bound(Some(AFT), PLEXUS, scope()).await,
18115 rig.bound(Some(BROCA), PLEXUS, scope()).await,
18116 rig.bound(Some(BROCA), OTHER, scope()).await,
18117 ];
18118 let mut record = session(1);
18119 (case.change)(&mut record);
18120 rig.sync(vec![record]).await;
18121 for (index, expected) in case.closed.iter().enumerate() {
18122 let route = &mut routes[index];
18123 match expected {
18124 Some(reason) => {
18125 assert!(!rig.live(route), "{}: route {index} still live", case.name);
18126 assert_eq!(
18127 route.closed_reason(),
18128 *reason,
18129 "{}: route {index}",
18130 case.name
18131 );
18132 }
18133 None => {
18134 assert!(rig.live(route), "{}: route {index} closed", case.name);
18135 assert!(
18136 route.untouched(),
18137 "{}: route {index} was told something",
18138 case.name
18139 );
18140 }
18141 }
18142 }
18143 }
18144 }
18145
18146 #[tokio::test]
18147 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
18148 for next in [Vec::new(), vec![session(2)]] {
18150 let mut rig = rig().await;
18151 rig.sync(vec![session(1)]).await;
18152 let mut route = rig
18153 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18154 .await;
18155 rig.sync(next).await;
18156 assert!(!rig.live(&route));
18157 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
18158 }
18159
18160 let mut rig = rig().await;
18163 let mut child = session(1);
18164 child.scope_ref = "child".to_string();
18165 child.kind = ScopeKind::Worker;
18166 child.parent = Some(ScopeParent::new(
18167 Principal::Reserved {
18168 module_id: OWNER.to_string(),
18169 },
18170 "s".to_string(),
18171 1,
18172 ));
18173 rig.sync(vec![session(1), child.clone()]).await;
18174 let mut child_route = rig
18175 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
18176 .await;
18177 assert_eq!(
18178 child_route.stamp().unwrap().parent_state,
18179 Some(ParentState::Linked)
18180 );
18181 rig.sync(vec![child]).await;
18182 assert!(!rig.live(&child_route));
18183 assert_eq!(
18184 child_route.closed_reason(),
18185 RouteCloseReason::ScopeParentEnded
18186 );
18187 }
18188
18189 #[tokio::test]
18190 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
18191 ) {
18192 let mut rig = rig().await;
18193 rig.sync(vec![session(1)]).await;
18194 let mut route = rig
18195 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18196 .await;
18197 let before = rig.forwarding.published_scope_tag(OWNER, "s");
18198 rig.sync(vec![session(1)]).await;
18199 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
18200 assert!(rig.live(&route) && route.untouched());
18201
18202 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
18206 .forwarding
18207 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18208 .unwrap()
18209 else {
18210 panic!("the route is bound");
18211 };
18212 binding.flow.acquire_tagged(9, false).await.unwrap();
18213 let mut widened = session(1);
18214 widened.carriers.push(carrier(MAGIC, None));
18215 rig.sync(vec![widened]).await;
18216 assert!(rig.live(&route) && route.untouched());
18217 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18218 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
18219 assert_eq!(binding.flow.in_flight(), 1);
18222 binding
18223 .flow
18224 .acquire_tagged(10, false)
18225 .await
18226 .expect("the flow is still open");
18227 }
18228
18229 #[tokio::test]
18232 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
18233 let mut rig = rig().await;
18234 rig.sync(vec![session(1)]).await;
18235 let mut on_incumbent = rig
18236 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18237 .await;
18238
18239 let (candidate, _candidate_rx) = wide_ctx(9);
18242 let registration = rig
18243 .handler
18244 .registry
18245 .register_candidate_with_control_ops(
18246 manifest(PLEXUS, PROTOCOL_VERSION),
18247 PROTOCOL_VERSION,
18248 candidate.connection_id,
18249 module_baseline_control_ops(),
18250 )
18251 .unwrap();
18252 rig.forwarding
18253 .register_candidate_module_connection(
18254 candidate.connection_id,
18255 PLEXUS.to_string(),
18256 PROTOCOL_VERSION,
18257 manifest_concurrency(®istration.manifest),
18258 candidate.egress.clone(),
18259 )
18260 .unwrap();
18261 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
18262 rig.handler
18263 .registry
18264 .promote_candidate(PLEXUS)
18265 .unwrap()
18266 .unwrap();
18267 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
18268
18269 rig.sync(Vec::new()).await;
18270 assert!(!rig.live(&on_incumbent));
18271 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
18272 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18273 let goodbye = incumbent_rx
18274 .try_recv()
18275 .expect("the superseded endpoint is told")
18276 .frame;
18277 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
18278 }
18279 }
18280}
18281
18282#[cfg(test)]
18283mod concurrency_default_exposure_tests {
18284 use super::*;
18285
18286 fn hello_body(role_json: &str) -> Vec<u8> {
18287 format!(
18288 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
18289 )
18290 .into_bytes()
18291 }
18292
18293 fn manifest_from(body: &[u8]) -> ModuleManifest {
18294 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
18295 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
18296 .expect("manifest parses")
18297 }
18298
18299 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
18300
18301 #[test]
18302 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
18303 let body = hello_body(&format!(
18304 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
18305 ));
18306 let manifest = manifest_from(&body);
18307 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18310 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
18311 }
18312
18313 #[test]
18314 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
18315 let body = hello_body(&format!(
18316 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
18317 ));
18318 let manifest = manifest_from(&body);
18319 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
18320 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
18321 }
18322
18323 #[test]
18324 fn non_management_roles_are_never_reported() {
18325 let body = hello_body(
18326 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
18327 );
18328 let manifest = manifest_from(&body);
18329 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
18330 }
18331}