1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6 fmt,
7 net::SocketAddr,
8 path::{Path, PathBuf},
9 sync::{Arc, Mutex, RwLock},
10 time::{Duration, Instant as StdInstant},
11};
12
13use serde::{Deserialize, Serialize};
14use subc_control::{
15 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
16 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
17 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
18 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
19 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
20 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
21 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
22};
23use subc_protocol::{
24 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
25 error_codes,
26 manifest::{
27 validate_hello_capability_grammar, validate_hello_confirm_operations,
28 validate_hello_display_name, validate_hello_event_declarations,
29 validate_hello_flow_operations, validate_hello_rich_ui_descriptor,
30 validate_hello_role_declarations, validate_hello_self_signal_declarations,
31 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
32 ProviderRole,
33 },
34 peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
35 scope::{
36 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
37 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
38 SCOPE_SYNC_OP,
39 },
40 session::{
41 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
42 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
43 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
44 ROLE_VERSIONS_FIELD,
45 },
46 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
47 Priority, RouteTarget, PROTOCOL_VERSION,
48};
49use tokio::time::{timeout_at, Instant};
50use tracing::{debug, info, instrument::WithSubscriber, warn};
51
52use crate::{
53 caller_identity::CallerIdentity,
54 capability_requirements::{
55 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
56 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
57 RequirementStatus, RuntimeModule,
58 },
59 child_roster::DaemonShutdownFlag,
60 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
61 forwarding::{
62 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
63 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
64 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
65 },
66 observability::{
67 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
68 },
69 privileged_prompt::{prompt_text, PrivilegedOperation},
70 provenance::{
71 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
72 },
73 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
74 router::{daemon_error_flags, RouteCtx, RouterError},
75 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
76 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
77 stderr_tail::{CaptureState, TailEntry},
78 supervise::{
79 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, ReservedHelloVerification,
80 SpawnSubscribeRefusal, SupervisorHandle, SwapHelloAdmission,
81 },
82 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
83};
84
85pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
91
92const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
93const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
94const CAP_PING_PONG: &str = "ping_pong_v1";
95const CAP_SESSION_ATTACH: &str = "session_attach_v1";
96const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
97
98const SUBC_CONTROL_OPS: &[&str] = &[
99 ops::DAEMON_SHUTDOWN_INTENT,
100 ops::ACTIVITY_RECORD,
101 ops::ACTIVITY_QUERY,
102 ops::ACTIVITY_SUBSCRIBE,
103 ops::SERVER_DESCRIBE,
104 ops::CATALOG_LIST,
105 ops::ROUTE_OPEN,
106 ops::ROUTE_POLL,
107 ops::ROUTE_CLOSING,
108 ops::ROUTE_CLOSED,
109 ops::SUPERVISOR_LIST,
110 ops::SUPERVISOR_RESTART,
111 ops::SUPERVISOR_SWAP,
112 ops::SUPERVISOR_RELOAD,
113 ops::SUPERVISOR_RESCAN,
114 ops::SUPERVISOR_RELEASE_RESERVED,
115 ops::SUPERVISOR_SET_ENABLED,
116 ops::SUPERVISOR_HEALTH_PROBE,
117 ops::SUPERVISOR_HEALTH,
118 ops::SUPERVISOR_STDERR_TAIL,
119 ops::SUPERVISOR_TERMINALS,
120 ops::SUPERVISOR_ROUTES,
121 ops::SUPERVISOR_PROVENANCE,
122 ops::SUPERVISOR_SPAWN_SNAPSHOT,
123 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
124];
125
126const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
133
134mod agent_commands;
135mod rescan_checked;
136use rescan_checked::{RescanMode, RescanPlan};
137
138const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
139 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
140 "supervisor.live_roots",
141 "supervisor.agent_commands",
142 SCOPE_SYNC_OP,
143 SCOPE_APPLY_OP,
144 SCOPE_DESCRIBE_OP,
145 "operator.confirm",
146];
147
148const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
152
153const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
154
155pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
163
164pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
179
180pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
198
199const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
200const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
201
202fn reload_verdict(
203 configured: &Path,
204 spawned_from: Option<&Path>,
205 image: subc_control::RunningImageAgreement,
206) -> PendingReloadVerdict {
207 let path = match spawned_from {
208 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
209 ReloadPathAgreement::Match
210 }
211 Some(spawned_from) => ReloadPathAgreement::Mismatch {
212 configured: configured.to_path_buf(),
213 spawned_from: spawned_from.to_path_buf(),
214 },
215 None => ReloadPathAgreement::Unavailable {
216 reason: if matches!(
217 image,
218 subc_control::RunningImageAgreement::Unavailable {
219 reason: subc_control::RunningImageUnavailableReason::NotRunning
220 }
221 ) {
222 ReloadPathUnavailableReason::NotRunning
223 } else {
224 ReloadPathUnavailableReason::SpawnedPathUnavailable
225 },
226 },
227 };
228 PendingReloadVerdict::new(path, image)
229}
230
231fn launch_spec_agreement(
232 configured: &crate::supervise::ModuleSpec,
233 running: bool,
234 spawned: Option<&crate::supervise::LaunchSpecDigest>,
235) -> subc_control::LaunchSpecAgreement {
236 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
237 if !running {
238 return Agreement::Unavailable {
239 reason: Reason::NotRunning,
240 };
241 }
242 match spawned {
243 None => Agreement::Unavailable {
244 reason: Reason::SpawnedSpecUnavailable,
245 },
246 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
247 Agreement::Match
248 }
249 Some(_) => Agreement::Mismatch,
250 }
251}
252
253fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
254 if configured == spawned_from {
255 return true;
256 }
257 #[cfg(windows)]
258 {
259 match (
265 std::fs::canonicalize(configured),
266 std::fs::canonicalize(spawned_from),
267 ) {
268 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
269 _ => false,
270 }
271 }
272 #[cfg(not(windows))]
273 {
274 false
275 }
276}
277
278#[derive(Clone)]
279struct DaemonProvenanceFacts {
280 build: DaemonBuildProvenance,
281 pid: Option<u32>,
282 started_at_ms: Option<u64>,
283 start_clock: Option<crate::clock::StartClock>,
284 executable_path: Option<PathBuf>,
285 executable_identity: Option<SpawnedFileIdentity>,
286 process_start_time: Option<u64>,
287 probe: ExecutableIdentityProbe,
288}
289
290impl Default for DaemonProvenanceFacts {
291 fn default() -> Self {
292 Self {
293 build: DaemonBuildProvenance {
294 build_git_sha: None,
295 build_lock_digest: None,
296 },
297 pid: None,
298 started_at_ms: None,
299 start_clock: None,
300 executable_path: None,
301 executable_identity: None,
302 process_start_time: None,
303 probe: ExecutableIdentityProbe::default(),
304 }
305 }
306}
307
308#[derive(Debug, Clone)]
309struct SupervisorRescanContext {
310 supervisor: Supervisor,
311 config_path: PathBuf,
312 configured_port: Option<u16>,
313 storage_config: Option<crate::daemon_config::StorageConfig>,
314 admission_facts_carrier_module_id: Option<String>,
315 admission_facts_targets: Option<Vec<String>>,
316 scope_authority_owners: Vec<String>,
317 federated_scope_owner: String,
318 privileged_operator_prompt: PrivilegedOperatorPrompt,
319 generation: Arc<std::sync::atomic::AtomicU64>,
325}
326
327const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
348 "reloading",
349 "supervisor_not_live",
350 "registration_not_active",
351 "no_forwarding_connection",
352 "relay_send_failed",
353];
354
355#[derive(Clone)]
357pub struct ControlHandler {
358 registry: Arc<Registry>,
359 activity: Option<crate::activity::ActivityLedger>,
360 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
361 forwarding: Arc<ForwardingTable>,
362 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
363 supervisor: SupervisorHandle,
364 pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
365 agent_declarations: agent_commands::Queues,
366 list_admission: Arc<tokio::sync::Semaphore>,
367 subc_capabilities: Arc<[String]>,
368 route_bind_relay_timeout: Duration,
372 route_bind_relay_timeouts: BTreeMap<String, Duration>,
376 route_bind_breakers: RouteBindBreakers,
379 route_bind_concurrency: RouteBindConcurrency,
382 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
387 route_bind_breaker_threshold: u32,
389 route_bind_breaker_cooldown: Duration,
391 health_probe_timeout: Duration,
392 storage_config: Option<crate::daemon_config::StorageConfig>,
395 machine_id: Option<crate::machine_id::MachineId>,
399 admission_facts_carrier_module_id: Option<String>,
400 admission_facts_targets: Option<Vec<String>>,
401 scopes: Arc<RwLock<ScopeTable>>,
405 scope_deadline_added: Arc<tokio::sync::Notify>,
409 scope_authority_owners: Vec<String>,
412 federated_scope_owner: String,
413 privileged_operator_prompt: PrivilegedOperatorPrompt,
415 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
418 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
419 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
420 local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
421 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
422 pub(crate) operator_apps:
423 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
424 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
429 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
435 #[cfg(test)]
437 pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
438 #[cfg(test)]
439 pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
440 #[allow(dead_code)]
444 #[cfg(windows)]
445 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
446 #[allow(dead_code)]
447 #[cfg(windows)]
448 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
449 #[allow(dead_code)]
453 #[cfg(windows)]
454 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
455 #[cfg(all(test, windows))]
458 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
459 #[cfg(test)]
460 caller_lookup: Option<TestCallerLookup>,
461 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
464 rescan: Option<SupervisorRescanContext>,
465 connected_clients: ConnectedClients,
466 counters: DaemonCounters,
467 local_endpoint_unavailable: Option<String>,
470 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
471 daemon_shutdown: DaemonShutdownFlag,
476 daemon_provenance: DaemonProvenanceFacts,
477 #[cfg(test)]
478 control_dispatch_delay: Option<Duration>,
479 #[cfg(test)]
480 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
481 #[cfg(test)]
485 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
486 #[cfg(test)]
492 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
493}
494
495impl fmt::Debug for ControlHandler {
496 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
497 f.debug_struct("ControlHandler")
498 .field("registry", &self.registry)
499 .field("forwarding", &self.forwarding)
500 .field("process_liveness", &self.process_liveness.is_some())
501 .field("supervisor", &self.supervisor)
502 .field("subc_capabilities", &self.subc_capabilities)
503 .finish()
504 }
505}
506
507struct RouteOpenRequest {
508 target: RouteTarget,
509 identity: BindIdentity,
510 consumer_identity: Option<ConsumerIdentity>,
511 consumer_capabilities: Option<Vec<String>>,
512 role_versions: Option<BTreeMap<String, String>>,
513 busy_refusals: Option<bool>,
514 admission_facts: Option<serde_json::Value>,
515 scope: Option<ScopeSelector>,
516}
517
518#[derive(Clone, Copy)]
519enum RouteOpenPhase {
520 Admission,
521 ProjectRoot,
522 EgressReservation,
523 BindSend,
524 BindAckWait,
525}
526
527struct RouteOpenTimings {
531 module_id: String,
532 principal: String,
533 admission: Duration,
534 project_root: Duration,
535 egress_reservation: Duration,
536 bind_send: Duration,
537 bind_ack_wait: Duration,
538 phase: Option<RouteOpenPhase>,
539 phase_started_at: Instant,
540}
541
542impl RouteOpenTimings {
543 fn new() -> Self {
544 Self {
545 module_id: String::new(),
546 principal: "unresolved".to_string(),
548 admission: Duration::ZERO,
549 project_root: Duration::ZERO,
550 egress_reservation: Duration::ZERO,
551 bind_send: Duration::ZERO,
552 bind_ack_wait: Duration::ZERO,
553 phase: Some(RouteOpenPhase::Admission),
554 phase_started_at: Instant::now(),
555 }
556 }
557
558 fn finish(&mut self) {
559 let elapsed = self.phase_started_at.elapsed();
560 match self.phase.take() {
561 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
562 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
563 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
564 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
565 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
566 None => {}
567 }
568 }
569
570 fn enter(&mut self, phase: RouteOpenPhase) {
571 self.finish();
572 self.phase = Some(phase);
573 self.phase_started_at = Instant::now();
574 }
575}
576
577struct RouteBindReservationGuard {
578 forwarding: Arc<ForwardingTable>,
579 endpoint: ModuleEndpointId,
580 relay_corr: u64,
581 armed: bool,
582}
583
584struct ModuleControlRpcGuard {
585 forwarding: Arc<ForwardingTable>,
586 endpoint: ModuleEndpointId,
587 corr: u64,
588 armed: bool,
589}
590
591impl ModuleControlRpcGuard {
592 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
593 Self {
594 forwarding,
595 endpoint,
596 corr,
597 armed: true,
598 }
599 }
600
601 fn disarm(&mut self) {
602 self.armed = false;
603 }
604}
605
606impl Drop for ModuleControlRpcGuard {
607 fn drop(&mut self) {
608 if self.armed {
609 let _ = self
610 .forwarding
611 .cancel_module_control_rpc(self.endpoint, self.corr);
612 }
613 }
614}
615
616impl RouteBindReservationGuard {
617 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
618 Self {
619 forwarding,
620 endpoint,
621 relay_corr,
622 armed: true,
623 }
624 }
625
626 fn release_and_disarm(&mut self) {
627 if !self.armed {
628 return;
629 }
630 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
631 self.endpoint,
632 self.relay_corr,
633 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
634 ) {
635 send_goodbye_target_best_effort(
636 &self.forwarding.counters(),
637 &target,
638 "canceled route.bind",
639 );
640 }
641 self.armed = false;
642 }
643
644 fn disarm(&mut self) {
645 self.armed = false;
646 }
647}
648
649impl Drop for RouteBindReservationGuard {
650 fn drop(&mut self) {
651 self.release_and_disarm();
652 }
653}
654
655#[derive(Debug, Clone, Default)]
685pub(crate) struct RouteBindBreakers {
686 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
687}
688
689#[derive(Debug, Clone, Default)]
690pub(crate) struct RouteBindConcurrency {
691 modules: Arc<Mutex<HashMap<String, usize>>>,
692}
693
694struct RouteBindConcurrencyGuard {
695 concurrency: RouteBindConcurrency,
696 module_id: String,
697}
698
699impl RouteBindConcurrency {
700 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
703 let mut modules = self
704 .modules
705 .lock()
706 .expect("route.bind concurrency mutex poisoned");
707 let in_flight = modules.entry(module_id.to_string()).or_default();
708 if *in_flight >= limit {
709 return Err(*in_flight);
710 }
711 *in_flight += 1;
712 Ok(RouteBindConcurrencyGuard {
713 concurrency: self.clone(),
714 module_id: module_id.to_string(),
715 })
716 }
717}
718
719impl Drop for RouteBindConcurrencyGuard {
720 fn drop(&mut self) {
721 let mut modules = self
722 .concurrency
723 .modules
724 .lock()
725 .expect("route.bind concurrency mutex poisoned");
726 let remove = {
727 let in_flight = modules
728 .get_mut(&self.module_id)
729 .expect("admitted route.bind has a concurrency entry");
730 *in_flight -= 1;
731 *in_flight == 0
732 };
733 if remove {
734 modules.remove(&self.module_id);
735 }
736 }
737}
738
739#[derive(Debug, Default)]
740struct ModuleBreakerState {
741 consecutive_timeouts: u32,
743 cooldown_until: Option<Instant>,
746 probe_in_flight: Option<Arc<()>>,
751}
752
753enum RouteBindAdmission<'a> {
755 Admitted {
756 guard: RouteBindBreakerGuard<'a>,
757 probe: bool,
760 },
761 Refused {
762 consecutive_timeouts: u32,
763 retry_in: Duration,
767 probe_in_flight: bool,
768 },
769}
770
771struct RouteBindBreakerGuard<'a> {
778 breakers: RouteBindBreakers,
779 module_id: &'a str,
780 probe_token: Option<Arc<()>>,
781 settled: bool,
782}
783
784impl RouteBindBreakerGuard<'_> {
785 fn record_accepted(&mut self) -> bool {
789 self.settled = true;
790 self.breakers.record_accepted(self.module_id)
791 }
792
793 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
796 self.settled = true;
797 self.breakers.record_timeout(
798 self.module_id,
799 self.probe_token.as_ref(),
800 threshold,
801 cooldown,
802 )
803 }
804
805 fn record_inconclusive(&mut self) {
814 self.settled = true;
815 self.breakers
816 .record_inconclusive(self.module_id, self.probe_token.as_ref());
817 }
818}
819
820impl Drop for RouteBindBreakerGuard<'_> {
821 fn drop(&mut self) {
822 if !self.settled {
823 self.breakers
824 .record_inconclusive(self.module_id, self.probe_token.as_ref());
825 }
826 }
827}
828
829struct BreakerOpened {
833 consecutive_timeouts: u32,
834 reopened_after_probe: bool,
837}
838
839impl RouteBindBreakers {
840 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
841 self.modules
842 .lock()
843 .expect("route.bind breaker mutex poisoned")
844 }
845
846 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
849 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
850 probe: probe_token.is_some(),
851 guard: RouteBindBreakerGuard {
852 breakers: self.clone(),
853 module_id,
854 probe_token,
855 settled: false,
856 },
857 };
858
859 let mut modules = self.lock();
860 let Some(state) = modules.get_mut(module_id) else {
861 return admitted(None);
862 };
863 let Some(cooldown_until) = state.cooldown_until else {
864 return admitted(None);
865 };
866 if state.probe_in_flight.is_some() {
867 return RouteBindAdmission::Refused {
868 consecutive_timeouts: state.consecutive_timeouts,
869 retry_in: Duration::ZERO,
870 probe_in_flight: true,
871 };
872 }
873 let now = Instant::now();
874 if now < cooldown_until {
875 return RouteBindAdmission::Refused {
876 consecutive_timeouts: state.consecutive_timeouts,
877 retry_in: cooldown_until - now,
878 probe_in_flight: false,
879 };
880 }
881 let token = Arc::new(());
882 state.probe_in_flight = Some(Arc::clone(&token));
883 admitted(Some(token))
884 }
885
886 fn record_accepted(&self, module_id: &str) -> bool {
887 self.lock()
888 .remove(module_id)
889 .is_some_and(|state| state.cooldown_until.is_some())
890 }
891
892 fn record_timeout(
893 &self,
894 module_id: &str,
895 probe_token: Option<&Arc<()>>,
896 threshold: u32,
897 cooldown: Duration,
898 ) -> Option<BreakerOpened> {
899 let mut modules = self.lock();
900 let state = modules.entry(module_id.to_string()).or_default();
901 let was_open = state.cooldown_until.is_some();
902 let was_probe = Self::owns_probe(state, probe_token);
903 if was_probe {
904 state.probe_in_flight = None;
905 }
906 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
907 if state.consecutive_timeouts < threshold {
908 return None;
909 }
910 state.cooldown_until = Some(Instant::now() + cooldown);
911 Some(BreakerOpened {
912 consecutive_timeouts: state.consecutive_timeouts,
913 reopened_after_probe: was_open && was_probe,
914 })
915 }
916
917 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
918 state
922 .probe_in_flight
923 .as_ref()
924 .zip(token)
925 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
926 }
927
928 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
929 if let Some(state) = self.lock().get_mut(module_id) {
930 if Self::owns_probe(state, probe_token) {
931 state.probe_in_flight = None;
932 }
933 }
934 }
935
936 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
951 self.lock()
952 .remove(module_id)
953 .map(|state| state.consecutive_timeouts)
954 .filter(|discarded| *discarded > 0)
955 }
956
957 fn open_snapshot(&self) -> Option<serde_json::Value> {
964 let now = Instant::now();
965 let modules = self.lock();
966 let open = modules
967 .iter()
968 .filter_map(|(module_id, state)| {
969 let cooldown_until = state.cooldown_until?;
970 Some((
971 module_id.clone(),
972 serde_json::json!({
973 "consecutive_timeouts": state.consecutive_timeouts,
974 "cooldown_remaining_ms":
975 cooldown_until.saturating_duration_since(now).as_millis() as u64,
976 "probe_in_flight": state.probe_in_flight.is_some(),
977 }),
978 ))
979 })
980 .collect::<serde_json::Map<String, serde_json::Value>>();
981 (!open.is_empty()).then_some(serde_json::Value::Object(open))
982 }
983}
984
985impl ControlHandler {
986 pub fn new(registry: Arc<Registry>) -> Self {
987 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
988 }
989
990 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
991 let counters = forwarding.counters();
992 let route_bind_breakers = forwarding.route_bind_breakers();
996 let route_bind_concurrency = forwarding.route_bind_concurrency();
997 let route_outages = forwarding.route_outages();
998 Self {
999 registry,
1000 activity: None,
1001 activity_streams: Arc::new(Mutex::new(HashMap::new())),
1002 forwarding,
1003 process_liveness: None,
1004 supervisor: SupervisorHandle::new(),
1005 resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1006 env!("CARGO_PKG_VERSION").to_owned(),
1007 )),
1008 agent_declarations: agent_commands::Queues::default(),
1009 list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1010 subc_capabilities: Arc::from([
1011 CAP_MANIFEST_REGISTRATION.to_string(),
1012 CAP_CHANNEL_LIFECYCLE.to_string(),
1013 CAP_PING_PONG.to_string(),
1014 CAP_SESSION_ATTACH.to_string(),
1015 CAP_ADMISSION_FACTS_RELAY.to_string(),
1016 CAP_SCOPES_V1.to_string(),
1017 subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY.to_string(),
1018 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1019 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1020 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1021 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1022 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1023 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1024 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1025 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1026 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1027 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1028 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1029 ]),
1030 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1031 route_bind_relay_timeouts: BTreeMap::new(),
1032 route_bind_breakers,
1033 route_bind_concurrency,
1034 route_outages,
1035 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1036 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1037 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1038 storage_config: None,
1039 machine_id: None,
1040 admission_facts_carrier_module_id: None,
1041 admission_facts_targets: None,
1042 scopes: Arc::new(RwLock::new(ScopeTable::new(
1043 crate::daemon_config::default_scope_authority_owners(),
1044 ))),
1045 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1046 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1047 federated_scope_owner: crate::daemon_config::default_federated_scope_owner(),
1048 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1049 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1050 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1051 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1052 local_connections: Arc::new(Mutex::new(HashSet::new())),
1053 operator_app_policy: Default::default(),
1054 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1055 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1056 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1057 #[cfg(test)]
1058 app_open_test_platform: None,
1059 #[cfg(test)]
1060 app_open_test_launch: None,
1061 #[cfg(windows)]
1062 windows_verify: None,
1063 #[cfg(windows)]
1064 windows_flash: None,
1065 #[cfg(windows)]
1066 windows_track: None,
1067 #[cfg(all(test, windows))]
1068 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1069 #[cfg(test)]
1070 caller_lookup: None,
1071 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1072 rescan: None,
1073 connected_clients: ConnectedClients::new(),
1074 local_endpoint_unavailable: None,
1075 counters,
1076 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1077 daemon_shutdown: DaemonShutdownFlag::default(),
1078 daemon_provenance: DaemonProvenanceFacts::default(),
1079 #[cfg(test)]
1080 control_dispatch_delay: None,
1081 #[cfg(test)]
1082 provenance_probe_override: None,
1083 #[cfg(test)]
1084 route_bind_relay_sent_hold: None,
1085 #[cfg(test)]
1086 hello_registry_to_forwarding_hold: None,
1087 }
1088 }
1089
1090 pub fn with_storage_config(
1093 mut self,
1094 storage_config: Option<crate::daemon_config::StorageConfig>,
1095 ) -> Self {
1096 self.storage_config = storage_config;
1097 self
1098 }
1099
1100 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1103 self.activity = Some(crate::activity::ActivityLedger::open(
1104 run_dir.join("activity.jsonl"),
1105 incarnation,
1106 ));
1107 self.supervisor.set_activity(self.activity.clone());
1108 self
1109 }
1110
1111 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1114 self.machine_id = machine_id;
1115 self
1116 }
1117
1118 pub fn with_admission_facts_config(
1122 mut self,
1123 carrier_module_id: Option<String>,
1124 targets: Option<Vec<String>>,
1125 ) -> Self {
1126 self.admission_facts_carrier_module_id = carrier_module_id;
1127 self.admission_facts_targets = targets;
1128 self
1129 }
1130
1131 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1135 self.scopes = Arc::new(RwLock::new(
1136 ScopeTable::new(owners.iter().cloned())
1137 .with_federated_owner(self.federated_scope_owner.clone()),
1138 ));
1139 self.scope_authority_owners = owners;
1140 self
1141 }
1142
1143 pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
1145 self.scopes = Arc::new(RwLock::new(
1146 ScopeTable::new(self.scope_authority_owners.iter().cloned())
1147 .with_federated_owner(owner.clone()),
1148 ));
1149 self.federated_scope_owner = owner;
1150 self
1151 }
1152
1153 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1155 self.privileged_operator_prompt = mode;
1156 self
1157 }
1158
1159 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1162 self.route_bind_relay_timeout = timeout;
1163 self
1164 }
1165
1166 pub fn with_route_bind_relay_timeouts(
1172 mut self,
1173 timeouts: impl IntoIterator<Item = (String, Duration)>,
1174 ) -> Self {
1175 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1176 self
1177 }
1178
1179 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1185 self.route_bind_relay_timeouts
1186 .get(module_id)
1187 .copied()
1188 .unwrap_or(self.route_bind_relay_timeout)
1189 }
1190
1191 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1199 self.route_bind_breaker_threshold = threshold.max(1);
1200 self.route_bind_breaker_cooldown = cooldown;
1201 self
1202 }
1203
1204 #[cfg(test)]
1205 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1206 self.health_probe_timeout = timeout;
1207 self
1208 }
1209
1210 #[cfg(test)]
1211 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1212 self.control_dispatch_delay = Some(delay);
1213 self
1214 }
1215
1216 #[cfg(test)]
1220 pub(crate) fn with_route_bind_relay_sent_hold(
1221 mut self,
1222 hold: Arc<tokio::sync::Barrier>,
1223 ) -> Self {
1224 self.route_bind_relay_sent_hold = Some(hold);
1225 self
1226 }
1227
1228 #[cfg(test)]
1233 pub(crate) fn with_hello_registry_to_forwarding_hold(
1234 mut self,
1235 hold: Arc<std::sync::Barrier>,
1236 ) -> Self {
1237 self.hello_registry_to_forwarding_hold = Some(hold);
1238 self
1239 }
1240
1241 #[cfg(test)]
1242 fn hold_between_registry_and_forwarding(&self) {
1243 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1244 hold.wait();
1245 hold.wait();
1246 }
1247 }
1248
1249 pub fn with_process_liveness(
1250 mut self,
1251 process_liveness: Arc<dyn ModuleProcessLiveness>,
1252 ) -> Self {
1253 self.process_liveness = Some(process_liveness);
1254 self
1255 }
1256
1257 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1258 self.supervisor = supervisor;
1259 self.supervisor.set_activity(self.activity.clone());
1260 self
1261 }
1262
1263 pub fn with_daemon_provenance(
1264 mut self,
1265 pid: u32,
1266 started_at_ms: u64,
1267 executable_path: Option<PathBuf>,
1268 build_git_sha: Option<String>,
1269 build_lock_digest: Option<String>,
1270 ) -> Self {
1271 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1272 let process_start_time = process_start_time(pid);
1273 self.daemon_provenance = DaemonProvenanceFacts {
1274 build: DaemonBuildProvenance {
1275 build_git_sha,
1276 build_lock_digest,
1277 },
1278 pid: Some(pid),
1279 started_at_ms: Some(started_at_ms),
1280 start_clock: None,
1281 executable_path,
1282 executable_identity,
1283 process_start_time,
1284 probe: ExecutableIdentityProbe::default(),
1285 };
1286 self
1287 }
1288
1289 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1290 self.daemon_provenance.start_clock = Some(clock);
1291 self
1292 }
1293
1294 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1296 self.scopes
1297 .write()
1298 .unwrap_or_else(|poisoned| poisoned.into_inner())
1299 .set_wall_clock(Arc::new(clock));
1300 self
1301 }
1302
1303 #[cfg(test)]
1304 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1305 self.provenance_probe_override = Some(result);
1306 self
1307 }
1308
1309 pub fn with_capability_config(
1313 self,
1314 modules: impl IntoIterator<Item = (String, bool)>,
1315 reserved_capabilities: BTreeMap<String, String>,
1316 ) -> Self {
1317 self.capability_evaluator
1318 .configure(modules, reserved_capabilities);
1319 self
1320 }
1321
1322 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1326 self.daemon_shutdown = flag;
1327 self
1328 }
1329
1330 pub fn with_supervisor_rescan(
1331 mut self,
1332 supervisor: Supervisor,
1333 config_path: impl Into<PathBuf>,
1334 configured_port: Option<u16>,
1335 ) -> Self {
1336 self.rescan = Some(SupervisorRescanContext {
1337 supervisor,
1338 config_path: config_path.into(),
1339 configured_port,
1340 storage_config: self.storage_config.clone(),
1341 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1342 admission_facts_targets: self.admission_facts_targets.clone(),
1343 scope_authority_owners: self.scope_authority_owners.clone(),
1344 federated_scope_owner: self.federated_scope_owner.clone(),
1345 privileged_operator_prompt: self.privileged_operator_prompt,
1346 generation: Arc::default(),
1347 });
1348 self
1349 }
1350
1351 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1352 self.connected_clients = connected_clients;
1353 self
1354 }
1355
1356 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1359 self.local_endpoint_unavailable = reason;
1360 self
1361 }
1362
1363 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1364 Arc::clone(&self.forwarding)
1365 }
1366
1367 pub(crate) fn counters(&self) -> DaemonCounters {
1368 self.counters.clone()
1369 }
1370
1371 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1374 tokio::spawn(async move {
1375 loop {
1376 self.capability_evaluator
1377 .wait_for_change_or_deadline()
1378 .await;
1379 self.refresh_capability_requirements();
1380 }
1381 });
1382 }
1383
1384 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1390 tokio::spawn(async move {
1391 loop {
1392 let has_deadlines = self
1393 .scopes
1394 .read()
1395 .unwrap_or_else(|poisoned| poisoned.into_inner())
1396 .has_deadlines();
1397 if !has_deadlines {
1398 self.scope_deadline_added.notified().await;
1399 continue;
1400 }
1401 tokio::time::sleep(Duration::from_secs(1)).await;
1402 if let Err(error) = self.sweep_expired_scopes() {
1403 warn!(%error, "scope expiry sweep failed");
1404 }
1405 }
1406 });
1407 }
1408
1409 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1410 let mut table = self
1411 .scopes
1412 .write()
1413 .unwrap_or_else(|poisoned| poisoned.into_inner());
1414 let swept = table.sweep_expired();
1415 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1416 drop(table);
1417 self.close_scope_drained_routes(drained);
1418 Ok(())
1419 }
1420
1421 fn publish_scope_changes(
1422 &self,
1423 changes: &[crate::scopes::ScopeTagChange],
1424 expired: &[crate::scopes::ScopeExpired],
1425 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1426 if changes.is_empty() {
1427 return Ok(Vec::new());
1428 }
1429 let drained = self
1430 .forwarding
1431 .publish_scope_changes(changes)
1432 .map_err(RouterError::Forwarding)?;
1433 let mut counts = HashMap::new();
1434 for route in &drained {
1435 *counts
1436 .entry((
1437 route.scope.owner.as_str(),
1438 route.scope.scope_ref.as_str(),
1439 route.scope.tag.scope_epoch,
1440 ))
1441 .or_insert(0usize) += 1;
1442 }
1443 for scope in expired {
1444 let routes_closed = counts
1445 .get(&(
1446 scope.owner.as_str(),
1447 scope.scope_ref.as_str(),
1448 scope.scope_epoch,
1449 ))
1450 .copied()
1451 .unwrap_or(0);
1452 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1453 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1454 routes_closed, "scope expired");
1455 }
1456 Ok(drained)
1457 }
1458
1459 fn runtime_capability_snapshot(
1460 &self,
1461 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1462 let runtime = self
1463 .supervisor
1464 .list()
1465 .into_iter()
1466 .map(|module| {
1467 let status = module.status().map_err(|err| {
1468 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1469 })?;
1470 Ok(RuntimeModule {
1471 module_id: status.module_id,
1472 state: status.state,
1473 enabled: status.enabled,
1474 })
1475 })
1476 .collect::<Result<Vec<_>, RouterError>>()?;
1477 let registrations = self.registry.list_registered_modules().map_err(|err| {
1483 RouterError::backend(
1484 0,
1485 0,
1486 format!("failed to list capability registrations: {err}"),
1487 )
1488 })?;
1489 let registrations = registrations
1490 .into_iter()
1491 .map(|registration| RegisteredModule {
1492 module_id: registration.manifest.module_id,
1493 module_version: registration.manifest.module_version,
1494 capabilities: registration.manifest.capabilities,
1495 })
1496 .collect();
1497 Ok((runtime, registrations))
1498 }
1499
1500 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1507 let cached_registration = RegisteredModule {
1508 module_id: registration.manifest.module_id.clone(),
1509 module_version: registration.manifest.module_version.clone(),
1510 capabilities: registration.manifest.capabilities.clone(),
1511 };
1512 let change = self.capability_evaluator.record_hello(&cached_registration);
1513 log_manifest_change(&cached_registration.module_id, &change);
1514 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1515 self.enforce_capability_denies();
1516 }
1517 self.refresh_capability_requirements();
1518 }
1519
1520 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1524 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1525 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1526 self.supervisor.set_swap_promotion_observer(observer);
1527 }
1528
1529 pub fn refresh_capability_requirements(&self) {
1530 match self.runtime_capability_snapshot() {
1531 Ok((runtime, registrations)) => {
1532 log_requirement_events(
1533 self.capability_evaluator
1534 .evaluate_now(&runtime, ®istrations),
1535 &self.daemon_shutdown,
1536 );
1537 }
1538 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1539 }
1540 }
1541
1542 fn enforce_capability_denies(&self) {
1546 let registrations = match self.registry.list_registered_modules() {
1547 Ok(snapshot) => snapshot,
1548 Err(err) => {
1549 warn!(error = %err, "failed to read registrations for capability deny census");
1550 return;
1551 }
1552 };
1553 let manifests = registrations
1554 .into_iter()
1555 .map(|registration| {
1556 (
1557 registration.manifest.module_id.clone(),
1558 registration.manifest,
1559 )
1560 })
1561 .collect::<BTreeMap<_, _>>();
1562 let census = match self.forwarding.route_census(None) {
1563 Ok(census) => census,
1564 Err(err) => {
1565 warn!(error = %err, "failed to read route census for capability deny enforcement");
1566 return;
1567 }
1568 };
1569
1570 for (target_module_id, routes) in census {
1571 let Some(target_manifest) = manifests.get(&target_module_id) else {
1572 continue;
1573 };
1574 let mut closed_routes = Vec::new();
1575 let mut goodbyes = Vec::new();
1576 for route in routes {
1577 let Principal::Reserved {
1578 module_id: opening_module_id,
1579 } = &route.principal
1580 else {
1581 continue;
1582 };
1583 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1584 continue;
1585 };
1586 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1587 continue;
1588 };
1589
1590 match self.forwarding.release_client_route(
1591 route.goodbye_target.connection_id,
1592 route.goodbye_target.channel,
1593 route.goodbye_target.epoch,
1594 ) {
1595 Ok(RouteRelease::Removed(module_goodbye)) => {
1596 warn!(
1597 opening_module_id,
1598 target_module_id,
1599 capability,
1600 "force-closing route because an attested capability deny edge now matches"
1601 );
1602 goodbyes.push(module_goodbye);
1603 goodbyes.push(route.goodbye_target.clone());
1604 closed_routes.push(route);
1605 }
1606 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1607 Err(err) => warn!(
1608 opening_module_id,
1609 target_module_id,
1610 capability,
1611 error = %err,
1612 "failed to force-close capability-denied route"
1613 ),
1614 }
1615 }
1616
1617 if closed_routes.is_empty() {
1618 continue;
1619 }
1620 send_route_control_pushes(
1621 &self.forwarding,
1622 closed_routes,
1623 ClientControlPush::RouteClosed {
1624 module_id: target_module_id,
1625 channels: Vec::new(),
1626 reason: RouteCloseReason::CapabilityDenied,
1627 drained: false,
1628 abandoned: 0,
1629 excluded_subscriptions: 0,
1630 terminal: Some(false),
1631 },
1632 );
1633 self.emit_route_goodbyes(goodbyes);
1636 }
1637 }
1638
1639 fn not_ready_reason(
1645 &self,
1646 registration: &crate::registry::ModuleRegistration,
1647 ) -> Option<NotReadyReason> {
1648 if !registration.ready {
1649 return Some(NotReadyReason {
1650 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1651 capability: None,
1652 });
1653 }
1654 self.first_unprovided_required_capability(registration)
1655 .map(|capability| NotReadyReason {
1656 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1657 capability: Some(capability),
1658 })
1659 }
1660
1661 fn first_unprovided_required_capability(
1679 &self,
1680 registration: &crate::registry::ModuleRegistration,
1681 ) -> Option<String> {
1682 let required = registration
1683 .manifest
1684 .capabilities
1685 .iter()
1686 .flat_map(|declarations| declarations.requires.iter())
1687 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1688 .map(|requirement| requirement.capability.as_str())
1689 .collect::<BTreeSet<_>>();
1690 if required.is_empty() {
1691 return None;
1692 }
1693 let module_id = registration.manifest.module_id.as_str();
1694 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1695 if required
1696 .iter()
1697 .any(|capability| verdict(capability).is_none())
1698 {
1699 self.refresh_capability_requirements();
1700 }
1701 required
1702 .into_iter()
1703 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1704 .map(str::to_string)
1705 }
1706
1707 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1708 self.capability_evaluator
1709 .statuses()
1710 .into_iter()
1711 .map(capability_requirement_status)
1712 .collect()
1713 }
1714
1715 fn deregister_connection(
1722 &self,
1723 connection_id: ConnectionId,
1724 reason: RegistrationEndReason,
1725 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1726 self.registry
1727 .deregister_connection_with_reason(connection_id, reason)
1728 }
1729
1730 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1731 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1732 return None;
1733 }
1734 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1735 parse_client_control_request(&frame.body)
1736 else {
1737 return None;
1738 };
1739 Some(target_module_id(&target).to_string())
1740 }
1741
1742 pub(crate) fn route_open_capacity_refusal(
1743 &self,
1744 ctx: &RouteCtx,
1745 frame: &Frame,
1746 target_module_id: &str,
1747 in_flight: usize,
1748 limit: usize,
1749 ) -> Result<Frame, RouterError> {
1750 let caller = self.route_open_caller_from_frame(frame);
1753 self.route_open_admission_refusal_frame(
1754 ctx,
1755 &caller,
1756 frame,
1757 target_module_id,
1758 "open_admission_full",
1759 (in_flight, limit),
1760 format!(
1761 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1762 ),
1763 )
1764 }
1765
1766 fn route_open_target_capacity_refusal(
1767 &self,
1768 ctx: &RouteCtx,
1769 caller: &RouteOpenCaller,
1770 frame: &Frame,
1771 target_module_id: &str,
1772 in_flight: usize,
1773 ) -> Result<Frame, RouterError> {
1774 self.route_open_admission_refusal_frame(
1775 ctx,
1776 caller,
1777 frame,
1778 target_module_id,
1779 "target_binds_full",
1780 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1781 format!(
1782 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1783 ),
1784 )
1785 }
1786
1787 #[allow(clippy::too_many_arguments)]
1795 fn route_open_admission_refusal_frame(
1796 &self,
1797 ctx: &RouteCtx,
1798 caller: &RouteOpenCaller,
1799 frame: &Frame,
1800 target_module_id: &str,
1801 reason: &'static str,
1802 (in_flight, limit): (usize, usize),
1803 message: impl Into<String>,
1804 ) -> Result<Frame, RouterError> {
1805 let code = error_codes::TARGET_UNAVAILABLE;
1806 self.counters.increment_route_open_refused(code);
1807 info!(
1808 target: "control",
1809 code,
1810 reason,
1811 module_id = ?target_module_id,
1812 connection_id = ctx.connection_id.get(),
1813 principal = %caller.principal,
1814 scope = %caller.scope,
1815 harness = %caller.harness,
1816 in_flight,
1817 limit,
1818 "route.open refused"
1819 );
1820 control_error_frame(frame, code, message.into())
1821 }
1822
1823 #[cfg(test)]
1829 pub fn handle_control(
1830 &self,
1831 connection_id: ConnectionId,
1832 frame: Frame,
1833 ) -> Result<Vec<Frame>, RouterError> {
1834 match frame.header.ty {
1835 FrameType::Ping => Ok(vec![pong(&frame)?]),
1836 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1837 FrameType::Goodbye => self.handle_goodbye(connection_id),
1838 ty => Ok(vec![control_error_frame(
1839 &frame,
1840 "unsupported_control_frame",
1841 format!("unsupported channel-0 frame {ty:?}"),
1842 )?]),
1843 }
1844 }
1845
1846 pub async fn handle_control_frame(
1847 &self,
1848 ctx: &RouteCtx,
1849 frame: Frame,
1850 ) -> Result<Vec<Frame>, RouterError> {
1851 self.handle_control_frame_timed(ctx, frame, None).await
1852 }
1853
1854 #[cfg(any(test, feature = "test-support"))]
1855 pub fn with_test_operator_app_policy(
1856 mut self,
1857 policy: crate::operator_apps::TestOperatorAppPolicy,
1858 ) -> Self {
1859 self.operator_app_policy = policy.into();
1860 self
1861 }
1862
1863 #[cfg(all(windows, any(test, feature = "test-support")))]
1867 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
1868 self.windows_verify = Some(verify);
1869 self
1870 }
1871
1872 #[cfg(all(windows, any(test, feature = "test-support")))]
1874 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
1875 self.windows_flash = Some(flash);
1876 self
1877 }
1878
1879 #[cfg(all(windows, any(test, feature = "test-support")))]
1883 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
1884 self.windows_track = Some(track);
1885 self
1886 }
1887
1888 pub(crate) fn record_captured_peer(
1889 &self,
1890 connection: ConnectionId,
1891 peer: subc_os::local_peer::CapturedPeer,
1892 ) {
1893 self.record_observed_peer(connection, peer.observation().clone());
1894 #[cfg(windows)]
1895 if let Some(stamp) = self.stamp_windows_launch(&peer) {
1896 self.operator_apps
1897 .lock()
1898 .unwrap_or_else(|p| p.into_inner())
1899 .insert(connection, stamp);
1900 }
1901 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1902 self.operator_apps
1903 .lock()
1904 .unwrap_or_else(|p| p.into_inner())
1905 .insert(connection, stamp);
1906 }
1907 }
1908
1909 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
1910 self.local_connections
1911 .lock()
1912 .unwrap_or_else(|p| p.into_inner())
1913 .insert(connection);
1914 }
1915
1916 fn connection_operator_app(
1919 &self,
1920 stamp: Option<&crate::operator_apps::OperatorAppStamp>,
1921 ) -> Option<subc_protocol::peer::OperatorApp> {
1922 stamp
1923 .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
1924 .map(|stamp| stamp.app.clone())
1925 }
1926
1927 fn handle_connection_describe(
1928 &self,
1929 ctx: &RouteCtx,
1930 frame: Frame,
1931 ) -> Result<Vec<Frame>, RouterError> {
1932 if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
1933 return Ok(vec![control_error_frame(
1934 &frame,
1935 "invalid_control_body",
1936 "malformed connection.describe body",
1937 )?]);
1938 }
1939 let local = self
1940 .local_connections
1941 .lock()
1942 .unwrap_or_else(|p| p.into_inner())
1943 .contains(&ctx.connection_id);
1944 let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
1945 let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
1946 Ok(vec![control_response_body_frame(
1947 &frame,
1948 &subc_control::ConnectionDescribeReply::new(local, operator_app),
1949 ops::CONNECTION_DESCRIBE,
1950 )?])
1951 }
1952
1953 pub(crate) fn record_observed_peer(
1954 &self,
1955 connection: ConnectionId,
1956 peer: subc_os::local_peer::ObservedPeer,
1957 ) {
1958 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1959 let peer = match peer {
1960 OsPeer::Unobserved => ObservedPeer::unobserved(),
1961 OsPeer::Observed(peer) => {
1962 let observed = match peer.user {
1963 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1964 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1965 };
1966 observed
1967 .with_exe(
1968 peer.exe
1969 .and_then(|path| path.into_os_string().into_string().ok()),
1970 )
1971 .with_signing_id(peer.signing_id)
1972 }
1973 };
1974 self.observed_peers
1975 .lock()
1976 .unwrap_or_else(|p| p.into_inner())
1977 .insert(connection, peer);
1978 }
1979
1980 pub(crate) fn record_caller_endpoints(
1981 &self,
1982 connection: ConnectionId,
1983 endpoints: Option<(SocketAddr, SocketAddr)>,
1984 ) {
1985 let mut callers = self
1986 .caller_endpoints
1987 .lock()
1988 .unwrap_or_else(|p| p.into_inner());
1989 callers.remove(&connection);
1990 if let Some(endpoints) = endpoints {
1991 callers.insert(connection, endpoints);
1992 }
1993 }
1994
1995 #[cfg(test)]
1996 pub(crate) fn with_caller_lookup_for_test(
1997 mut self,
1998 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1999 ) -> Self {
2000 self.caller_lookup = Some(Arc::new(lookup));
2001 self
2002 }
2003
2004 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
2005 let endpoints = self
2006 .caller_endpoints
2007 .lock()
2008 .unwrap_or_else(|p| p.into_inner())
2009 .get(&connection)
2010 .copied();
2011 #[cfg(test)]
2012 let lookup = self.caller_lookup.clone();
2013 tokio::task::spawn_blocking(move || {
2014 #[cfg(test)]
2015 if let Some(lookup) = lookup {
2016 return lookup(endpoints);
2017 }
2018 endpoints
2019 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2020 .unwrap_or_default()
2021 })
2022 .await
2023 .unwrap_or_default()
2024 }
2025
2026 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2027 frame.header.channel == 0
2028 && frame.header.ty == FrameType::Request
2029 && self
2030 .forwarding
2031 .module_endpoint_for_connection(ctx.connection_id)
2032 .is_ok_and(|endpoint| endpoint.is_none())
2033 && parse_client_control_request(&frame.body)
2034 .ok()
2035 .is_some_and(|request| privileged_operation(&request).is_some())
2036 }
2037
2038 pub(crate) async fn handle_control_frame_timed(
2039 &self,
2040 ctx: &RouteCtx,
2041 frame: Frame,
2042 dispatch_started_at: Option<StdInstant>,
2043 ) -> Result<Vec<Frame>, RouterError> {
2044 match frame.header.ty {
2045 FrameType::Ping => Ok(vec![pong(&frame)?]),
2046 FrameType::Hello => {
2047 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
2048 }
2049 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2050 FrameType::Cancel => {
2051 if self
2054 .agent_declarations
2055 .cancel(ctx.connection_id, frame.header.corr)
2056 || self
2057 .forwarding
2058 .operator_confirms()
2059 .cancel(ctx.connection_id, frame.header.corr)
2060 || self
2061 .supervisor
2062 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2063 || self.cancel_activity_subscription(ctx, &frame)?
2064 {
2065 Ok(Vec::new())
2066 } else {
2067 Ok(vec![control_error_frame(
2068 &frame,
2069 "unknown_subscription",
2070 "no subscription has this correlation id",
2071 )?])
2072 }
2073 }
2074 FrameType::Request => {
2075 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2078 if op
2079 .as_ref()
2080 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2081 {
2082 return self.handle_activity_record(ctx, frame).await;
2083 }
2084 if op.as_ref().is_some_and(|probe| {
2085 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2086 }) {
2087 return self.handle_activity_read(ctx, frame).await;
2088 }
2089 if op
2090 .as_ref()
2091 .is_some_and(|probe| probe.op == "operator.confirm")
2092 {
2093 return self.handle_operator_confirm(ctx, frame);
2094 }
2095 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2096 return self
2097 .observe_control_wait(
2098 ctx,
2099 None,
2100 ops::APP_OPEN,
2101 self.handle_app_open(ctx, frame),
2102 )
2103 .await;
2104 }
2105 if op
2106 .as_ref()
2107 .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2108 {
2109 return self.handle_connection_describe(ctx, frame);
2110 }
2111 if op
2112 .as_ref()
2113 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2114 {
2115 return self.handle_scope_served_by(ctx.connection_id, frame);
2116 }
2117 if self
2118 .forwarding
2119 .module_endpoint_for_connection(ctx.connection_id)
2120 .map_err(RouterError::Forwarding)?
2121 .is_some()
2122 {
2123 if !is_known_module_request_op(&frame.body) {
2124 return Ok(vec![control_error_frame(
2125 &frame,
2126 "unsupported_control_frame",
2127 "module-originated channel-0 REQUEST is not supported",
2128 )?]);
2129 }
2130 let request = match parse_module_control_request_from_module(&frame.body) {
2131 Ok(request) => request,
2132 Err((err, ControlRequestBodyError::UnknownOp)) => {
2133 return Ok(vec![control_error_frame(
2134 &frame,
2135 "unsupported_control_frame",
2136 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2137 )?])
2138 }
2139 Err((err, ControlRequestBodyError::InvalidBody)) => {
2140 return Ok(vec![control_error_frame(
2141 &frame,
2142 "invalid_control_body",
2143 format!("malformed module control body: {err}"),
2144 )?])
2145 }
2146 };
2147 let op = module_control_request_op(&request);
2148 let corr = frame.header.corr;
2149 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2150 if let ModuleControlRequestFromModule::AgentCommands {
2151 started,
2152 finished,
2153 known,
2154 } = request
2155 {
2156 return self
2157 .agent_declarations
2158 .enqueue(
2159 self,
2160 ctx,
2161 frame,
2162 crate::resource_totals::Declaration {
2163 started,
2164 finished,
2165 known,
2166 },
2167 )
2168 .await;
2169 }
2170 let result =
2171 self.handle_module_control_request(ctx.connection_id, frame, request);
2172 log_slow_control_dispatch(
2173 dispatch_started_at,
2174 op,
2175 ctx.connection_id,
2176 corr,
2177 None,
2178 );
2179 return result;
2180 }
2181
2182 if is_known_module_request_op(&frame.body) {
2183 let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2184 .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2185 return Ok(vec![control_error_frame(
2186 &frame,
2187 if is_agents { "not_supervised" } else { "not_registered" },
2188 "module-originated control requests require the module's own registered connection",
2189 )?]);
2190 }
2191
2192 let request = match parse_client_control_request(&frame.body) {
2193 Ok(request) => request,
2194 Err((err, ControlRequestBodyError::UnknownOp)) => {
2195 return Ok(vec![control_error_frame(
2196 &frame,
2197 "unknown_control_op",
2198 format!("unknown client control op: {err}"),
2199 )?])
2200 }
2201 Err((err, ControlRequestBodyError::InvalidBody)) => {
2202 return Ok(vec![control_error_frame(
2203 &frame,
2204 "invalid_control_body",
2205 format!("malformed client control body: {err}"),
2206 )?])
2207 }
2208 };
2209 let op = client_control_request_op(&request);
2210 let corr = frame.header.corr;
2211 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2212 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2213 #[cfg(test)]
2214 if let Some(delay) = self.control_dispatch_delay {
2215 tokio::time::sleep(delay).await;
2216 }
2217 let result = self
2218 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2219 .await;
2220 if let Some(timings) = route_open_timings.as_mut() {
2221 timings.finish();
2222 }
2223 log_slow_control_dispatch(
2224 dispatch_started_at,
2225 op,
2226 ctx.connection_id,
2227 corr,
2228 route_open_timings.as_ref(),
2229 );
2230 result
2231 }
2232 FrameType::Push => {
2233 let Some(endpoint) = self
2234 .forwarding
2235 .module_endpoint_for_connection(ctx.connection_id)
2236 .map_err(RouterError::Forwarding)?
2237 else {
2238 return Ok(vec![control_error_frame(
2239 &frame,
2240 "unsupported_control_frame",
2241 "client-originated channel-0 PUSH is not supported",
2242 )?]);
2243 };
2244 self.handle_status_update(endpoint, frame)
2245 }
2246 FrameType::Response | FrameType::Error
2247 if self
2248 .forwarding
2249 .module_endpoint_for_connection(ctx.connection_id)
2250 .map_err(RouterError::Forwarding)?
2251 .is_some() =>
2252 {
2253 self.handle_module_relay_response(ctx.connection_id, frame)
2254 }
2255 ty => Ok(vec![control_error_frame(
2256 &frame,
2257 "unsupported_control_frame",
2258 format!("unsupported channel-0 frame {ty:?}"),
2259 )?]),
2260 }
2261 }
2262
2263 pub fn cleanup_connection(
2264 &self,
2265 connection_id: ConnectionId,
2266 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2267 self.local_connections
2268 .lock()
2269 .unwrap_or_else(|p| p.into_inner())
2270 .remove(&connection_id);
2271 self.caller_endpoints
2272 .lock()
2273 .unwrap_or_else(|p| p.into_inner())
2274 .remove(&connection_id);
2275 self.build_class_warned
2276 .lock()
2277 .unwrap_or_else(|p| p.into_inner())
2278 .remove(&connection_id);
2279 self.observed_peers
2280 .lock()
2281 .unwrap_or_else(|p| p.into_inner())
2282 .remove(&connection_id);
2283 self.operator_apps
2284 .lock()
2285 .unwrap_or_else(|p| p.into_inner())
2286 .remove(&connection_id);
2287 self.forwarding
2288 .operator_confirms()
2289 .connection_closed(connection_id);
2290 self.cleanup_connection_with_end_reason(
2291 connection_id,
2292 RegistrationEndReason::ConnectionClosed,
2293 )
2294 }
2295
2296 fn cleanup_connection_with_end_reason(
2297 &self,
2298 connection_id: ConnectionId,
2299 requested_reason: RegistrationEndReason,
2300 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2301 self.agent_declarations.close(connection_id);
2302 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2303 self.registry
2304 .get_module_by_connection(connection_id)?
2305 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2306 .and_then(|module| module.registration_end_reason().ok().flatten())
2307 .unwrap_or(requested_reason)
2308 } else {
2309 requested_reason
2310 };
2311 let crash_closed = self
2312 .registry
2313 .get_module_by_connection(connection_id)?
2314 .and_then(|registration| {
2315 self.forwarding
2316 .module_endpoint_for_connection(connection_id)
2317 .ok()
2318 .flatten()
2319 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2320 .map(|routes| (registration.manifest.module_id, routes))
2321 });
2322 let crash_closed = crash_closed.map(|(module_id, routes)| {
2323 let terminal = match self.supervisor.get(&module_id) {
2324 None => false,
2325 Some(module) => match module.will_recover_after_connection_loss() {
2326 Ok(will_recover) => !will_recover,
2327 Err(err) => {
2328 warn!(
2329 %module_id,
2330 error = %err,
2331 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2332 );
2333 false
2334 }
2335 },
2336 };
2337 let (reason, drained) = match self
2341 .forwarding
2342 .daemon_drain_result_for_connection(connection_id)
2343 {
2344 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2345 Ok(None) => (RouteCloseReason::Crash, false),
2346 Err(err) => {
2347 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2348 (RouteCloseReason::Crash, false)
2349 }
2350 };
2351 (module_id, routes, reason, drained, terminal)
2352 });
2353 let registrations = self.deregister_connection(connection_id, end_reason);
2354 let cleanup = if crash_closed.is_some() {
2355 self.forwarding.cleanup_connection_counted(connection_id)
2356 } else {
2357 self.forwarding
2363 .cleanup_connection(connection_id)
2364 .map(|released| crate::forwarding::ConnectionCleanup {
2365 released,
2366 abandoned_relays: 0,
2367 })
2368 };
2369 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2374 let abandoned = cleanup
2375 .as_ref()
2376 .map_or(0, |cleanup| cleanup.abandoned_relays);
2377 send_route_control_pushes(
2378 &self.forwarding,
2379 routes,
2380 ClientControlPush::RouteClosed {
2381 module_id,
2382 channels: Vec::new(),
2383 reason,
2384 drained,
2385 abandoned,
2386 excluded_subscriptions: 0,
2387 terminal: Some(terminal),
2388 },
2389 );
2390 }
2391 if let Ok(cleanup) = cleanup {
2392 self.emit_route_goodbyes(cleanup.released);
2393 }
2394 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2398 crate::supervise::notify_registration_release();
2399 self.capability_evaluator.wake_deadline_loop();
2400 self.refresh_capability_requirements();
2401 }
2402 self.supervisor.remove_spawn_subscribers(connection_id);
2403 self.activity_streams
2404 .lock()
2405 .unwrap_or_else(|p| p.into_inner())
2406 .retain(|(id, _), task| {
2407 if *id == connection_id {
2408 task.abort();
2409 false
2410 } else {
2411 true
2412 }
2413 });
2414 self.hello_launch_nonces
2417 .lock()
2418 .unwrap_or_else(|poisoned| poisoned.into_inner())
2419 .forget(connection_id);
2420 self.scopes
2421 .write()
2422 .unwrap_or_else(|poisoned| poisoned.into_inner())
2423 .release_connection(connection_id);
2424 registrations
2425 }
2426
2427 pub(crate) fn handle_route_goodbye(
2428 &self,
2429 connection_id: ConnectionId,
2430 route_channel: u16,
2431 route_epoch: u32,
2432 ) -> Result<bool, RouterError> {
2433 debug!(
2434 connection_id = connection_id.get(),
2435 route_channel, route_epoch, "handling route GOODBYE"
2436 );
2437 let RouteRelease::Removed(released_route) = self
2438 .forwarding
2439 .release_client_route(connection_id, route_channel, route_epoch)
2440 .map_err(RouterError::Forwarding)?
2441 else {
2442 return Ok(false);
2443 };
2444 self.emit_route_goodbyes(vec![released_route]);
2445 Ok(true)
2446 }
2447
2448 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2449 for released in released_routes {
2450 let frame = match Frame::build_with_version(
2451 released.negotiated_ver,
2452 FrameType::Goodbye,
2453 control_flags(),
2454 released.channel,
2455 released.epoch,
2456 0,
2457 Vec::new(),
2458 ) {
2459 Ok(frame) => frame,
2460 Err(err) => {
2461 warn!(
2462 route_channel = released.channel,
2463 error = %err,
2464 "failed to build route GOODBYE frame"
2465 );
2466 continue;
2467 }
2468 };
2469 if !released.close_on_delivery_failure() {
2470 crate::forwarding::send_module_route_goodbye(
2471 &self.counters,
2472 &released.sink,
2473 frame,
2474 released.module_id.as_deref(),
2475 "client route released",
2476 );
2477 continue;
2478 }
2479 if let Err(err) = released.sink.try_send(frame) {
2480 warn!(
2481 target_connection_id = released.connection_id.get(),
2482 route_channel = released.channel,
2483 error = %err,
2484 "route GOODBYE was not delivered to client; closing target connection"
2485 );
2486 if self
2487 .forwarding
2488 .escalate_client_delivery_failure(
2489 released.connection_id,
2490 released.channel,
2491 released.epoch,
2492 CloseReason::new(
2493 "route_goodbye_delivery_failed",
2494 format!(
2495 "failed to enqueue route GOODBYE for channel {}: {err}",
2496 released.channel
2497 ),
2498 ),
2499 crate::forwarding::UndeliveredFrame {
2500 module_id: released.module_id.as_deref(),
2501 sink: &released.sink,
2502 },
2503 )
2504 .unwrap_or(false)
2505 {
2506 self.counters.increment_goodbye_relay_client_failed();
2507 }
2508 }
2509 }
2510 }
2511
2512 fn send_abandoned_route_bind_goodbye(
2524 &self,
2525 module_sink: &crate::FrameSink,
2526 negotiated_ver: u8,
2527 module_channel: u16,
2528 module_epoch: u32,
2529 ) {
2530 let frame = match Frame::build_with_version(
2531 negotiated_ver,
2532 FrameType::Goodbye,
2533 control_flags(),
2534 module_channel,
2535 module_epoch,
2536 0,
2537 Vec::new(),
2538 ) {
2539 Ok(frame) => frame,
2540 Err(err) => {
2541 warn!(
2542 route_channel = module_channel,
2543 error = %err,
2544 "failed to build GOODBYE for abandoned route.bind"
2545 );
2546 return;
2547 }
2548 };
2549 crate::forwarding::send_module_route_goodbye(
2550 &self.counters,
2551 module_sink,
2552 frame,
2553 None,
2554 "abandoned route.bind",
2555 );
2556 }
2557
2558 fn handle_hello(
2559 &self,
2560 connection_id: ConnectionId,
2561 sink: Option<crate::FrameSink>,
2562 frame: Frame,
2563 ) -> Result<Vec<Frame>, RouterError> {
2564 debug!(
2565 connection_id = connection_id.get(),
2566 corr = frame.header.corr,
2567 "handling HELLO"
2568 );
2569 if self
2573 .registry
2574 .get_module_by_connection(connection_id)
2575 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2576 .is_some()
2577 {
2578 return Ok(vec![control_error_frame(
2579 &frame,
2580 "invalid_hello",
2581 "connection is already registered as a module",
2582 )?]);
2583 }
2584 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2585 Ok(value) => value,
2586 Err(err) => {
2587 return Ok(vec![control_error_frame(
2588 &frame,
2589 "invalid_hello",
2590 format!("malformed HELLO body: {err}"),
2591 )?])
2592 }
2593 };
2594 if let Err(err) = validate_hello_display_name(&hello_value) {
2595 return Ok(vec![control_error_frame(
2596 &frame,
2597 "invalid_manifest",
2598 err.to_string(),
2599 )?]);
2600 }
2601 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2602 return Ok(vec![control_error_frame(
2603 &frame,
2604 "invalid_capability_grammar",
2605 err.to_string(),
2606 )?]);
2607 }
2608 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2609 return Ok(vec![control_error_frame(
2610 &frame,
2611 "invalid_manifest",
2612 err.to_string(),
2613 )?]);
2614 }
2615 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2616 return Ok(vec![control_error_frame(
2617 &frame,
2618 "invalid_event_declaration",
2619 err.to_string(),
2620 )?]);
2621 }
2622 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2623 return Ok(vec![control_error_frame(
2624 &frame,
2625 "invalid_manifest",
2626 err.to_string(),
2627 )?]);
2628 }
2629 if let Some(provenance) = hello_value
2630 .get("manifest")
2631 .and_then(|manifest| manifest.get("provenance"))
2632 {
2633 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2634 return Ok(vec![control_error_frame(
2635 &frame,
2636 "invalid_manifest",
2637 format!("malformed manifest provenance: {err}"),
2638 )?]);
2639 }
2640 }
2641 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2642 let module_id = hello_value["manifest"]["module_id"]
2643 .as_str()
2644 .unwrap_or("<unknown>");
2645 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2646 }
2647 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2648 let module_id = hello_value["manifest"]["module_id"]
2649 .as_str()
2650 .unwrap_or("<unknown>");
2651 warn!(module_id, %reason, "dropping flow_operations from HELLO");
2652 }
2653 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2654 let module_id = hello_value["manifest"]["module_id"]
2655 .as_str()
2656 .unwrap_or("<unknown>");
2657 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2658 }
2659 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2660 Ok(hello) => hello,
2661 Err(err) => {
2662 return Ok(vec![control_error_frame(
2663 &frame,
2664 "invalid_hello",
2665 format!("malformed HELLO body: {err}"),
2666 )?])
2667 }
2668 };
2669
2670 if hello.protocol_ver != hello.manifest.protocol_ver {
2671 return Ok(vec![control_error_frame(
2672 &frame,
2673 "invalid_manifest",
2674 format!(
2675 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2676 hello.protocol_ver, hello.manifest.protocol_ver
2677 ),
2678 )?]);
2679 }
2680
2681 if hello.manifest.module_id.trim().is_empty() {
2682 return Ok(vec![control_error_frame(
2683 &frame,
2684 "invalid_manifest",
2685 "manifest module_id must not be empty",
2686 )?]);
2687 }
2688
2689 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2690 Ok(negotiated_ver) => negotiated_ver,
2691 Err(message) => {
2692 return Ok(vec![control_error_frame(
2693 &frame,
2694 "version_unsupported",
2695 message,
2696 )?])
2697 }
2698 };
2699
2700 let swap_admission = self
2708 .supervisor
2709 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2710 if swap_admission == SwapHelloAdmission::Refused {
2711 warn!(
2712 module_id = %hello.manifest.module_id,
2713 connection_id = connection_id.get(),
2714 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2715 );
2716 return Ok(vec![control_error_frame(
2717 &frame,
2718 "swap_token_invalid",
2719 format!(
2720 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2721 hello.manifest.module_id
2722 ),
2723 )?]);
2724 }
2725 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2726
2727 let verification = if swap_candidate {
2732 ReservedHelloVerification {
2733 nonce_verified: true,
2734 prefix_owner: None,
2735 }
2736 } else {
2737 match self.supervisor.reserved_hello_verification(
2738 &hello.manifest.module_id,
2739 hello.launch_nonce.as_deref(),
2740 ) {
2741 Ok(verified) => verified,
2742 Err(rejection) => {
2743 let message = match rejection {
2744 ReservedHelloRejection::Exact { module_id } => format!(
2745 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2746 ),
2747 ReservedHelloRejection::Prefix {
2748 prefix,
2749 owner_module_id,
2750 } => format!(
2751 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2752 hello.manifest.module_id
2753 ),
2754 };
2755 return Ok(vec![control_error_frame(
2756 &frame,
2757 "reserved_module",
2758 message,
2759 )?]);
2760 }
2761 }
2762 };
2763
2764 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2765 &hello.manifest.module_id,
2766 hello.manifest.capabilities.as_ref(),
2767 );
2768 if let Some(refusal) = reserved_capability_refusals.first() {
2769 let capability = refusal.capability.clone();
2770 let bound_module = refusal.claimants[0].clone();
2771 log_duplicate_claim_events(reserved_capability_refusals);
2772 return Ok(vec![control_error_frame(
2773 &frame,
2774 "reserved_capability",
2775 format!(
2776 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2777 capability, bound_module, hello.manifest.module_id
2778 ),
2779 )?]);
2780 }
2781
2782 if self
2785 .forwarding
2786 .connection_has_client_routes(connection_id)
2787 .map_err(RouterError::Forwarding)?
2788 {
2789 return Ok(vec![control_error_frame(
2790 &frame,
2791 "invalid_hello",
2792 "connection has open client routes and cannot also register as a module",
2793 )?]);
2794 }
2795
2796 self.hello_launch_nonces
2801 .lock()
2802 .unwrap_or_else(|poisoned| poisoned.into_inner())
2803 .record(connection_id, hello.launch_nonce.as_deref());
2804 let control_ops = effective_module_control_ops(hello.control_ops);
2805 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2808 if swap_candidate {
2809 return self.register_swap_candidate(
2810 connection_id,
2811 sink,
2812 &frame,
2813 hello.manifest,
2814 negotiated_ver,
2815 control_ops,
2816 hello_ack,
2817 );
2818 }
2819 let registration = match self
2826 .registry
2827 .register_unpublished_verified_with_control_ops(
2828 hello.manifest,
2829 negotiated_ver,
2830 connection_id,
2831 control_ops,
2832 verification.nonce_verified,
2833 ) {
2834 Ok(registration) => registration,
2835 Err(RegistryError::DuplicateModuleId { module_id }) => {
2836 return Ok(vec![control_error_frame(
2837 &frame,
2838 "duplicate_module_id",
2839 format!(
2840 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2841 ),
2842 )?])
2843 }
2844 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2845 return Ok(vec![control_error_frame(
2846 &frame,
2847 "invalid_module_id",
2848 err.to_string(),
2849 )?])
2850 }
2851 Err(err) => {
2852 return Ok(vec![control_error_frame(
2853 &frame,
2854 "registry_error",
2855 err.to_string(),
2856 )?])
2857 }
2858 };
2859 #[cfg(test)]
2860 self.hold_between_registry_and_forwarding();
2861
2862 let reply = if let Some(sink) = sink {
2863 let concurrency = manifest_concurrency(®istration.manifest);
2879 if let Err(err) = self.forwarding.register_module_connection_acked(
2880 connection_id,
2881 registration.manifest.module_id.clone(),
2882 negotiated_ver,
2883 concurrency,
2884 sink,
2885 crate::forwarding::ModuleHelloAck {
2886 frame: hello_ack,
2887 verified_prefix_owner: verification.prefix_owner,
2888 },
2889 ) {
2890 if matches!(
2895 self.deregister_connection(
2896 connection_id,
2897 RegistrationEndReason::RegistrationFailed,
2898 ),
2899 Ok(r) if !r.is_empty()
2900 ) {
2901 crate::supervise::notify_registration_release();
2902 }
2903 return Ok(vec![control_error_frame(
2904 &frame,
2905 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2906 "invalid_hello"
2907 } else {
2908 forwarding_error_code(&err)
2909 },
2910 err.to_string(),
2911 )?]);
2912 }
2913 Vec::new()
2914 } else {
2915 vec![hello_ack]
2918 };
2919
2920 if !self
2925 .registry
2926 .publish_registration(connection_id)
2927 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2928 {
2929 debug!(
2930 module_id = %registration.manifest.module_id,
2931 connection_id = connection_id.get(),
2932 "registration was removed before it could be published"
2933 );
2934 }
2935
2936 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2945 info!(
2946 module_id = %registration.manifest.module_id,
2947 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2948 );
2949 }
2950
2951 self.apply_registration_capabilities(®istration);
2952
2953 self.supervisor
2954 .record_activity_registration(®istration.manifest.module_id);
2955
2956 info!(
2957 module_id = %registration.manifest.module_id,
2958 module_version = %registration.manifest.module_version,
2959 negotiated_ver,
2960 routable_provider = manifest_provides_routable_role(®istration.manifest),
2961 connection_id = connection_id.get(),
2962 "module registered"
2963 );
2964
2965 Ok(reply)
2966 }
2967
2968 #[allow(clippy::too_many_arguments)]
2985 fn register_swap_candidate(
2986 &self,
2987 connection_id: ConnectionId,
2988 sink: Option<crate::FrameSink>,
2989 frame: &Frame,
2990 manifest: ModuleManifest,
2991 negotiated_ver: u8,
2992 control_ops: Vec<String>,
2993 hello_ack: Frame,
2994 ) -> Result<Vec<Frame>, RouterError> {
2995 let module_id = manifest.module_id.clone();
2996 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2997 manifest,
2998 negotiated_ver,
2999 connection_id,
3000 control_ops,
3001 ) {
3002 Ok(registration) => registration,
3003 Err(RegistryError::DuplicateModuleId { module_id }) => {
3004 return Ok(vec![control_error_frame(
3005 frame,
3006 "duplicate_module_id",
3007 format!(
3008 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
3009 ),
3010 )?])
3011 }
3012 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
3013 return Ok(vec![control_error_frame(
3014 frame,
3015 "invalid_module_id",
3016 err.to_string(),
3017 )?])
3018 }
3019 Err(err) => {
3020 return Ok(vec![control_error_frame(
3021 frame,
3022 "registry_error",
3023 err.to_string(),
3024 )?])
3025 }
3026 };
3027 #[cfg(test)]
3028 self.hold_between_registry_and_forwarding();
3029 let reply = if let Some(sink) = sink {
3030 let concurrency = manifest_concurrency(®istration.manifest);
3034 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
3035 connection_id,
3036 module_id.clone(),
3037 negotiated_ver,
3038 concurrency,
3039 sink,
3040 hello_ack,
3041 ) {
3042 if matches!(
3043 self.deregister_connection(
3044 connection_id,
3045 RegistrationEndReason::RegistrationFailed,
3046 ),
3047 Ok(r) if !r.is_empty()
3048 ) {
3049 crate::supervise::notify_registration_release();
3050 }
3051 return Ok(vec![control_error_frame(
3052 frame,
3053 forwarding_error_code(&err),
3054 err.to_string(),
3055 )?]);
3056 }
3057 Vec::new()
3058 } else {
3059 vec![hello_ack]
3060 };
3061 if !self
3062 .registry
3063 .publish_registration(connection_id)
3064 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3065 {
3066 debug!(
3067 module_id = %module_id,
3068 connection_id = connection_id.get(),
3069 "swap candidate registration was removed before it could be published"
3070 );
3071 }
3072 self.supervisor.mark_swap_candidate_admitted(&module_id);
3073 info!(
3074 module_id = %module_id,
3075 module_version = %registration.manifest.module_version,
3076 negotiated_ver,
3077 ready = registration.ready,
3078 connection_id = connection_id.get(),
3079 "swap candidate registered; not routable until cutover"
3080 );
3081 Ok(reply)
3082 }
3083
3084 fn build_hello_ack(
3085 &self,
3086 frame: &Frame,
3087 negotiated_ver: u8,
3088 module_id: &str,
3089 ) -> Result<Frame, RouterError> {
3090 let ack = ModuleHelloAckBody {
3091 negotiated_ver,
3092 subc_ops: module_subc_ops(),
3093 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
3094 storage: self
3095 .storage_config
3096 .as_ref()
3097 .map(|cfg| cfg.descriptor_for(module_id)),
3098 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3099 };
3100 let body = serde_json::to_vec(&ack).map_err(|err| {
3101 RouterError::backend(
3102 0,
3103 frame.header.corr,
3104 format!("failed to encode HELLO_ACK: {err}"),
3105 )
3106 })?;
3107
3108 Frame::build_with_version(
3109 negotiated_ver,
3110 FrameType::HelloAck,
3111 control_flags(),
3112 0,
3113 0,
3114 frame.header.corr,
3115 body,
3116 )
3117 .map_err(RouterError::FrameBuild)
3118 }
3119
3120 async fn handle_client_control_request(
3121 &self,
3122 ctx: &RouteCtx,
3123 frame: Frame,
3124 request: ClientControlRequest,
3125 route_open_timings: &mut Option<RouteOpenTimings>,
3126 ) -> Result<Vec<Frame>, RouterError> {
3127 if let Some(operation) = privileged_operation(&request) {
3128 let caller = self.resolve_caller(ctx.connection_id).await;
3129 let op = client_control_request_op(&request);
3130 let text = prompt_text(operation, &caller);
3131 match self.privileged_operator_prompt {
3132 PrivilegedOperatorPrompt::Log => {
3133 info!(target: "subc_daemon::privileged_prompt", operation = op,
3134 target = ?operation, caller = %caller.description(), summary = %text,
3135 "privileged request would prompt the operator");
3136 }
3137 PrivilegedOperatorPrompt::Enforce => {
3138 let key =
3139 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
3140 let stamp = self
3141 .operator_apps
3142 .lock()
3143 .unwrap_or_else(|p| p.into_inner())
3144 .get(&ctx.connection_id)
3145 .cloned();
3146 let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
3147 &self.operator_app_policy,
3148 &self.launch_table,
3149 stamp.as_ref(),
3150 )
3151 .then(|| self.shortcut_context(stamp, None, None));
3152 let outcome = self
3153 .forwarding
3154 .operator_confirms()
3155 .approve_privileged_with_shortcut(
3156 ctx,
3157 &frame,
3158 op,
3159 text,
3160 caller.description(),
3161 key,
3162 shortcut,
3163 )
3164 .await;
3165 if outcome.code != "confirmed" {
3166 return Ok(vec![outcome.frame(&frame)]);
3167 }
3168 }
3169 }
3170 }
3171 match request {
3172 ClientControlRequest::DaemonShutdownIntent { intent } => {
3173 if intent.as_str() != Some("restart") {
3174 return Ok(vec![control_error_frame(
3175 &frame,
3176 "invalid_params",
3177 "intent must be 'restart'",
3178 )?]);
3179 }
3180 let response = ClientControlResponse::DaemonShutdownIntent {
3181 intent: "restart".to_owned(),
3182 expires_at_ms: self.supervisor.announce_daemon_restart(),
3183 };
3184 Ok(vec![control_response_body_frame(
3185 &frame,
3186 &response,
3187 "ClientControlResponse::DaemonShutdownIntent",
3188 )?])
3189 }
3190 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
3191 ClientControlRequest::CatalogList { module_id } => {
3192 self.handle_catalog_list(frame, module_id)
3193 }
3194 ClientControlRequest::RouteOpen {
3195 target,
3196 identity,
3197 consumer_identity,
3198 consumer_capabilities,
3199 role_versions,
3200 busy_refusals,
3201 admission_facts,
3202 scope,
3203 ..
3204 } => {
3205 self.handle_route_open(
3206 ctx,
3207 frame,
3208 RouteOpenRequest {
3209 target,
3210 identity,
3211 consumer_identity,
3212 consumer_capabilities,
3213 role_versions,
3214 busy_refusals,
3215 admission_facts,
3216 scope,
3217 },
3218 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
3219 )
3220 .await
3221 }
3222 ClientControlRequest::RoutePoll {
3223 route_channel,
3224 route_epoch,
3225 kind,
3226 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
3227 ClientControlRequest::SupervisorList {} => {
3228 let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
3229 self.agent_declarations.refuse(
3230 self,
3231 ctx,
3232 control_error_frame(
3233 &frame,
3234 "busy",
3235 "supervisor list work is at capacity; retry",
3236 )?,
3237 );
3238 return Ok(Vec::new());
3239 };
3240 let handler = self.clone();
3242 let sink = ctx.egress.clone();
3243 tokio::spawn(
3244 async move {
3245 let _permit = permit;
3246 let result = handler.handle_supervisor_list(frame.clone()).await;
3247 let replies = match result {
3248 Ok(replies) => replies,
3249 Err(error) => {
3250 control_error_frame(&frame, "backend_error", error.to_string())
3251 .into_iter()
3252 .collect()
3253 }
3254 };
3255 for reply in replies {
3256 if !matches!(
3257 tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
3258 .await,
3259 Ok(Ok(()))
3260 ) {
3261 break;
3262 }
3263 }
3264 }
3265 .with_current_subscriber(),
3266 );
3267 Ok(Vec::new())
3268 }
3269 ClientControlRequest::SupervisorSpawnSnapshot {} => {
3270 self.handle_supervisor_spawn_snapshot(frame)
3271 }
3272 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
3273 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
3274 }
3275 ClientControlRequest::SupervisorRestart {
3276 module_id,
3277 drain_timeout_ms,
3278 } => {
3279 self.log_supervisor_request_received(
3280 ctx,
3281 frame.header.corr,
3282 ops::SUPERVISOR_RESTART,
3283 Some(&module_id),
3284 None,
3285 )?;
3286 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3287 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
3288 .await
3289 }
3290 ClientControlRequest::SupervisorSwap {
3291 module_id,
3292 ready_timeout_ms,
3293 } => {
3294 self.log_supervisor_request_received(
3295 ctx,
3296 frame.header.corr,
3297 ops::SUPERVISOR_SWAP,
3298 Some(&module_id),
3299 None,
3300 )?;
3301 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3302 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3303 .await
3304 }
3305 ClientControlRequest::SupervisorReload { module_id } => {
3306 self.log_supervisor_request_received(
3307 ctx,
3308 frame.header.corr,
3309 ops::SUPERVISOR_RELOAD,
3310 Some(&module_id),
3311 None,
3312 )?;
3313 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3314 self.handle_supervisor_reload(frame, module_id, requester)
3315 .await
3316 }
3317 ClientControlRequest::SupervisorRescan { preview } => {
3318 if !preview {
3319 self.log_supervisor_request_received(
3320 ctx,
3321 frame.header.corr,
3322 ops::SUPERVISOR_RESCAN,
3323 None,
3324 None,
3325 )?;
3326 }
3327 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3328 let operation = self.handle_supervisor_rescan(
3329 frame,
3330 if preview {
3331 RescanMode::Preview
3332 } else {
3333 RescanMode::Execute
3334 },
3335 requester,
3336 );
3337 self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
3338 .await
3339 }
3340 ClientControlRequest::SupervisorRescanChecked {
3341 preview,
3342 candidate,
3343 expect,
3344 } => {
3345 self.observe_control_wait(
3346 ctx,
3347 None,
3348 ops::SUPERVISOR_RESCAN,
3349 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
3350 )
3351 .await
3352 }
3353 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3354 self.log_supervisor_request_received(
3355 ctx,
3356 frame.header.corr,
3357 ops::SUPERVISOR_RELEASE_RESERVED,
3358 Some(&module_id),
3359 None,
3360 )?;
3361 self.handle_supervisor_release_reserved(frame, module_id)
3362 .await
3363 }
3364 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3365 self.log_supervisor_request_received(
3366 ctx,
3367 frame.header.corr,
3368 ops::SUPERVISOR_SET_ENABLED,
3369 Some(&module_id),
3370 Some(enabled),
3371 )?;
3372 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3373 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3374 .await
3375 }
3376 ClientControlRequest::SupervisorHealthProbe { module_id } => {
3377 self.handle_supervisor_health_probe(ctx, frame, module_id)
3378 .await
3379 }
3380 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3381 ClientControlRequest::SupervisorRoutes { module_id } => {
3382 self.handle_supervisor_routes(frame, module_id)
3383 }
3384 ClientControlRequest::SupervisorProvenance { module_id } => {
3385 self.handle_supervisor_provenance(ctx, frame, module_id)
3386 .await
3387 }
3388 ClientControlRequest::SupervisorStderrTail {
3389 module_id,
3390 max_lines,
3391 max_bytes,
3392 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3393 ClientControlRequest::SupervisorTerminals { module_id } => {
3394 self.handle_supervisor_terminals(ctx, frame, module_id)
3395 .await
3396 }
3397 }
3398 }
3399
3400 fn log_supervisor_request_received(
3401 &self,
3402 ctx: &RouteCtx,
3403 corr: u64,
3404 op: &'static str,
3405 module_id: Option<&str>,
3406 enabled: Option<bool>,
3407 ) -> Result<(), RouterError> {
3408 let caller = self.supervisor_requester(ctx, corr)?;
3409 let caller = principal_label(&caller);
3410 info!(
3413 target: "control",
3414 op,
3415 module_id,
3416 enabled,
3417 connection_id = ctx.connection_id.get(),
3418 caller = %caller,
3419 "supervisor request received"
3420 );
3421 Ok(())
3422 }
3423
3424 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3425 Ok(self
3426 .registry
3427 .get_module_by_connection(ctx.connection_id)
3428 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3429 .map(|registration| Principal::Reserved {
3430 module_id: registration.manifest.module_id,
3431 })
3432 .unwrap_or(Principal::Direct))
3433 }
3434
3435 fn handle_module_control_request(
3436 &self,
3437 connection_id: ConnectionId,
3438 frame: Frame,
3439 request: ModuleControlRequestFromModule,
3440 ) -> Result<Vec<Frame>, RouterError> {
3441 match request {
3442 ModuleControlRequestFromModule::CatalogUpdate {
3443 provides,
3444 capabilities,
3445 ready,
3446 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3447 ModuleControlRequestFromModule::LiveRoots {} => {
3448 let registered = self
3449 .registry
3450 .get_module_by_connection(connection_id)
3451 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3452 let Some(registration) = registered else {
3453 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3454 };
3455 let response = self
3456 .forwarding
3457 .live_roots(®istration.manifest.module_id)
3458 .map_err(RouterError::Forwarding)?;
3459 Ok(vec![control_response_body_frame(
3460 &frame,
3461 &response,
3462 "ModuleControlResponseToModule::LiveRoots",
3463 )?])
3464 }
3465 ModuleControlRequestFromModule::AgentCommands {
3466 started,
3467 finished,
3468 known,
3469 } => self.handle_agent_commands(connection_id, frame, started, finished, known),
3470 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3471 self.handle_scope_sync(connection_id, frame, generation, scopes)
3472 }
3473 ModuleControlRequestFromModule::ScopeApply {
3474 generation,
3475 upsert,
3476 end,
3477 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3478 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3479 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3480 }
3481 _ => Ok(vec![control_error_frame(
3482 &frame,
3483 "unsupported_control_frame",
3484 "unsupported module control operation",
3485 )?]),
3486 }
3487 }
3488
3489 fn handle_scope_served_by(
3490 &self,
3491 connection_id: ConnectionId,
3492 frame: Frame,
3493 ) -> Result<Vec<Frame>, RouterError> {
3494 let registered = self
3495 .registry
3496 .get_module_by_connection(connection_id)
3497 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3498 let Some(registered) = registered else {
3499 return Ok(vec![control_error_frame(
3500 &frame,
3501 "not_registered",
3502 "scope.served_by requires an active module registration owned by this connection",
3503 )?]);
3504 };
3505 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3506 {
3507 Ok(request) => request,
3508 Err(err) => {
3509 return Ok(vec![control_error_frame(
3510 &frame,
3511 "invalid_control_body",
3512 format!("malformed scope.served_by body: {err}"),
3513 )?])
3514 }
3515 };
3516 let answer = self
3517 .scopes
3518 .read()
3519 .unwrap_or_else(|poisoned| poisoned.into_inner())
3520 .served_by(
3521 ®istered.manifest.module_id,
3522 &request.scope_ref,
3523 request.scope_epoch,
3524 &request.module_id,
3525 );
3526 let reply = subc_control::ScopeServedByReply::new(
3527 answer,
3528 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3529 );
3530 Ok(vec![control_response_body_frame(
3531 &frame,
3532 &reply,
3533 "ScopeServedByReply",
3534 )?])
3535 }
3536
3537 fn handle_operator_confirm(
3538 &self,
3539 ctx: &RouteCtx,
3540 frame: Frame,
3541 ) -> Result<Vec<Frame>, RouterError> {
3542 use crate::operator_confirm::{audit, Outcome};
3543 let registration = self
3544 .registry
3545 .get_module_by_connection(ctx.connection_id)
3546 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3547 let Some(registration) = registration else {
3548 let outcome = Outcome::refusal("not_registered");
3549 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3550 return Ok(vec![outcome.frame(&frame)]);
3551 };
3552 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3553 Ok(request) => request,
3554 Err(_) => {
3555 let outcome = Outcome::refusal("invalid_control_body");
3556 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3557 return Ok(vec![outcome.frame(&frame)]);
3558 }
3559 };
3560 let declared = matches!(
3561 request.kind,
3562 Some(subc_protocol::session::OperatorConfirmKind::Operation)
3563 ) && request.operation.as_ref().is_some_and(|operation| {
3564 registration
3565 .manifest
3566 .confirm_operations
3567 .as_ref()
3568 .is_some_and(|operations| operations.contains(operation))
3569 });
3570 let module_id = registration.manifest.module_id;
3571 let nonce = self
3575 .hello_launch_nonces
3576 .lock()
3577 .unwrap_or_else(|p| p.into_inner())
3578 .nonce(ctx.connection_id)
3579 .map(str::to_owned);
3580 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3581 self.supervisor
3582 .spawned_consumer_authorized(&module_id, nonce)
3583 });
3584 let confirms = self.forwarding.operator_confirms();
3585 self.forwarding
3586 .with_operator_route(
3587 ctx.connection_id,
3588 request.route_channel,
3589 request.route_epoch,
3590 |binding| {
3591 let shortcut = binding
3592 .filter(|binding| {
3593 nonce_proven
3594 && crate::operator_confirm::valid_summary(&request.summary)
3595 && declared
3596 && crate::operator_confirm::ShortcutContext::route_eligible(
3597 &self.operator_app_policy,
3598 &self.launch_table,
3599 binding,
3600 )
3601 })
3602 .map(|binding| {
3603 let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
3604 &self.operator_app_policy,
3605 binding,
3606 );
3607 let stamp = if phone.is_some() {
3608 None
3609 } else {
3610 binding.operator_app.clone()
3611 };
3612 self.shortcut_context(
3613 stamp,
3614 phone,
3615 Some((request.route_channel, request.route_epoch)),
3616 )
3617 });
3618 confirms.admit_with_shortcut(
3619 ctx,
3620 frame,
3621 module_id,
3622 nonce_proven,
3623 request,
3624 binding,
3625 shortcut,
3626 )
3627 },
3628 )
3629 .map_err(RouterError::Forwarding)
3630 }
3631
3632 fn shortcut_context(
3633 &self,
3634 stamp: Option<crate::operator_apps::OperatorAppStamp>,
3635 phone_fingerprint: Option<String>,
3636 route: Option<(u16, u32)>,
3637 ) -> crate::operator_confirm::ShortcutContext {
3638 crate::operator_confirm::ShortcutContext {
3639 ledger: self.activity.clone(),
3640 forwarding: self.forwarding.clone(),
3641 launches: self.launch_table.clone(),
3642 stamp,
3643 phone_fingerprint,
3644 route,
3645 connections: self.operator_apps.clone(),
3646 #[cfg(test)]
3647 hooks: Default::default(),
3648 }
3649 }
3650
3651 #[cfg(any(test, feature = "test-support"))]
3653 pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
3654 self.operator_app_policy.phone_enabled = enabled;
3655 self
3656 }
3657
3658 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3661 let mut scopes = ScopeRecords::from(scopes);
3662 for (index, record) in scopes.records.iter_mut().enumerate() {
3663 if let Some(refusal) = Self::canonicalize_workdir(record) {
3664 scopes.refusals.insert(index, refusal);
3665 }
3666 }
3667 scopes
3668 }
3669
3670 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3671 let workdir = record.workdir.as_mut()?;
3672 let path = std::path::Path::new(workdir);
3673 if !path.is_absolute() {
3674 return Some((
3675 error_codes::INVALID_WORKDIR,
3676 "workdir must be absolute".to_string(),
3677 ));
3678 }
3679 match ProjectRootId::from_path_allowing_missing(path) {
3680 Ok(root) => match root.as_path().to_str() {
3681 Some(canonical) => {
3682 *workdir = canonical.to_string();
3683 None
3684 }
3685 None => Some((
3686 error_codes::INVALID_WORKDIR,
3687 "canonical workdir is not UTF-8".to_string(),
3688 )),
3689 },
3690 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3691 }
3692 }
3693
3694 fn handle_scope_sync(
3698 &self,
3699 connection_id: ConnectionId,
3700 frame: Frame,
3701 generation: u64,
3702 scopes: Vec<ScopeRecord>,
3703 ) -> Result<Vec<Frame>, RouterError> {
3704 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3705 }
3706
3707 fn handle_scope_change(
3708 &self,
3709 connection_id: ConnectionId,
3710 frame: Frame,
3711 generation: u64,
3712 scopes: Vec<ScopeRecord>,
3713 end: Option<Vec<ScopeEnd>>,
3714 ) -> Result<Vec<Frame>, RouterError> {
3715 let is_apply = end.is_some();
3716 let Some(registration) = self
3717 .registry
3718 .get_module_by_connection(connection_id)
3719 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3720 else {
3721 return Ok(vec![control_error_frame(
3722 &frame,
3723 "not_registered",
3724 "scope.sync and scope.apply require an active module registration owned by this connection",
3725 )?]);
3726 };
3727 let owner = registration.manifest.module_id;
3728 let scopes = Self::prepare_scope_records(scopes);
3729 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3730 let is_current_launch = |connection: ConnectionId| {
3731 self.hello_launch_nonces
3732 .lock()
3733 .unwrap_or_else(|poisoned| poisoned.into_inner())
3734 .presented(connection, current_nonce.as_deref())
3735 };
3736 let mut table = self
3741 .scopes
3742 .write()
3743 .unwrap_or_else(|poisoned| poisoned.into_inner());
3744 let outcome = match end {
3745 Some(end) => table.apply(
3746 &owner,
3747 connection_id,
3748 is_current_launch,
3749 generation,
3750 scopes,
3751 end,
3752 ),
3753 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3754 };
3755 let drained = match &outcome {
3756 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3757 Err(_) => Vec::new(),
3758 };
3759 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3760 drop(table);
3761 if has_deadlines {
3762 self.scope_deadline_added.notify_one();
3763 }
3764 match outcome {
3765 Ok(applied) => {
3766 let counts = ScopeOutcomeCounts::of(&applied.results);
3767 info!(
3768 owner = %owner,
3769 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3770 generation,
3771 records = applied.results.len(),
3772 created = counts.created,
3773 replaced = counts.replaced,
3774 updated = counts.updated,
3775 unchanged = counts.unchanged,
3776 refused = counts.refused,
3777 ended = applied.ended.len(),
3778 tag_changes = applied.tag_changes.len(),
3779 routes_closed = drained.len(),
3780 "scope change accepted"
3781 );
3782 for refused in applied
3788 .results
3789 .iter()
3790 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3791 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3792 {
3793 warn!(
3794 owner = %owner,
3795 generation,
3796 scope_ref = %refused.scope_ref,
3797 scope_epoch = refused.scope_epoch,
3798 code = refused.code.as_deref().unwrap_or(""),
3799 "scope record refused"
3800 );
3801 }
3802 self.close_scope_drained_routes(drained);
3803 let response = if is_apply {
3804 ModuleControlResponseToModule::ScopeApply {
3805 generation,
3806 results: applied.results,
3807 end_results: applied.end_results,
3808 ended: applied.ended,
3809 }
3810 } else {
3811 ModuleControlResponseToModule::ScopeSync {
3812 generation,
3813 results: applied.results,
3814 ended: applied.ended,
3815 }
3816 };
3817 Ok(vec![control_response_body_frame(
3818 &frame,
3819 &response,
3820 "scope change response",
3821 )?])
3822 }
3823 Err(refusal) => {
3824 info!(
3825 owner = %owner,
3826 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3827 generation,
3828 code = refusal.code,
3829 "scope change refused"
3830 );
3831 Ok(vec![control_error_frame(
3832 &frame,
3833 refusal.code,
3834 refusal.message,
3835 )?])
3836 }
3837 }
3838 }
3839
3840 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3846 if drained.is_empty() {
3847 return;
3848 }
3849 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3850 BTreeMap::new();
3851 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3852 for route in drained {
3853 warn!(
3854 module_id = %route.module_id,
3855 reason = ?route.reason,
3856 client_connection_id = route.client.connection_id.get(),
3857 route_channel = route.client.channel,
3858 "closing route because its scope changed"
3859 );
3860 pushes
3861 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3862 .or_insert_with(|| (route.reason, Vec::new()))
3863 .1
3864 .push(EndpointRoute {
3865 goodbye_target: route.client.clone(),
3866 principal: Principal::Unverified,
3867 bound_at: Instant::now(),
3868 draining: false,
3869 drain_reason: None,
3870 });
3871 goodbyes.push(route.module);
3872 goodbyes.push(route.client);
3873 }
3874 for ((module_id, _), (reason, routes)) in pushes {
3875 send_route_control_pushes(
3876 &self.forwarding,
3877 routes,
3878 ClientControlPush::RouteClosed {
3879 module_id,
3880 channels: Vec::new(),
3881 reason,
3882 drained: false,
3883 abandoned: 0,
3884 excluded_subscriptions: 0,
3885 terminal: Some(false),
3886 },
3887 );
3888 }
3889 self.emit_route_goodbyes(goodbyes);
3890 }
3891
3892 fn handle_scope_describe(
3895 &self,
3896 connection_id: ConnectionId,
3897 frame: Frame,
3898 owner: Principal,
3899 scope_ref: String,
3900 ) -> Result<Vec<Frame>, RouterError> {
3901 let registered = self
3902 .registry
3903 .get_module_by_connection(connection_id)
3904 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3905 let Some(registered) = registered else {
3906 return Ok(vec![control_error_frame(
3907 &frame,
3908 "not_registered",
3909 "scope.describe requires an active module registration owned by this connection",
3910 )?]);
3911 };
3912 let scopes = self
3913 .scopes
3914 .read()
3915 .unwrap_or_else(|poisoned| poisoned.into_inner());
3916 if scopes.is_federated_owner(&owner)
3917 && !crate::scopes::target_supports_federated(registered.manifest.capabilities.as_ref())
3918 {
3919 return Ok(vec![control_error_frame(
3920 &frame,
3921 error_codes::SCOPE_FEDERATED_UNSUPPORTED,
3922 "scope.describe of the federated owner requires scope-federated/v1",
3923 )?]);
3924 }
3925 let mut description = scopes.describe(&owner, &scope_ref);
3926 drop(scopes);
3927 if !registered
3930 .manifest
3931 .capabilities
3932 .as_ref()
3933 .is_some_and(|caps| {
3934 caps.provides
3935 .iter()
3936 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3937 })
3938 {
3939 if let Some(stamp) = description.stamp.as_mut() {
3940 stamp.attributes.on_behalf_of_agent = None;
3941 }
3942 }
3943 if !registered
3944 .manifest
3945 .capabilities
3946 .as_ref()
3947 .is_some_and(|caps| {
3948 caps.provides
3949 .iter()
3950 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3951 })
3952 {
3953 if let Some(stamp) = description.stamp.as_mut() {
3954 stamp.attributes.flow_version = None;
3955 stamp.attributes.flow_code_hash = None;
3956 stamp.attributes.flow_install_id = None;
3957 }
3958 }
3959 let owner_configured = match &owner {
3960 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3965 _ => false,
3966 };
3967 let response = ModuleControlResponseToModule::ScopeDescribe {
3968 status: description.status,
3969 scope_epoch: description.scope_epoch,
3970 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3971 owner_synced: description.owner_synced,
3972 owner_configured,
3973 scope: description.stamp,
3974 };
3975 Ok(vec![control_response_body_frame(
3976 &frame,
3977 &response,
3978 "ModuleControlResponseToModule::ScopeDescribe",
3979 )?])
3980 }
3981
3982 fn handle_catalog_update(
3983 &self,
3984 connection_id: ConnectionId,
3985 frame: Frame,
3986 provides: Vec<ProviderRole>,
3987 capabilities: Option<CapabilityDeclarations>,
3988 ready: Option<bool>,
3989 ) -> Result<Vec<Frame>, RouterError> {
3990 self.refresh_capability_requirements();
3991 let Some(registration) = self
3992 .registry
3993 .get_module_by_connection(connection_id)
3994 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3995 else {
3996 return Ok(vec![control_error_frame(
3997 &frame,
3998 "not_registered",
3999 "catalog.update requires an active module registration owned by this connection",
4000 )?]);
4001 };
4002
4003 if let Some(message) =
4004 catalog_update_frozen_field_message(®istration.manifest, &provides)
4005 {
4006 return Ok(vec![control_error_frame(
4007 &frame,
4008 "catalog_update_frozen_field",
4009 message,
4010 )?]);
4011 }
4012
4013 let mut candidate = registration.manifest.clone();
4014 candidate.provides = provides.clone();
4015 candidate.capabilities = capabilities
4016 .clone()
4017 .or_else(|| registration.manifest.capabilities.clone());
4018 if let Some(rich_ui) = &candidate.rich_ui {
4019 if let Err(err) = rich_ui.validate() {
4020 return Ok(vec![control_error_frame(
4021 &frame,
4022 "invalid_manifest",
4023 err.to_string(),
4024 )?]);
4025 }
4026 }
4027 if let Err(err) = candidate.validate_capability_grammar() {
4028 return Ok(vec![control_error_frame(
4029 &frame,
4030 "invalid_capability_grammar",
4031 err.to_string(),
4032 )?]);
4033 }
4034
4035 let mut conflicts = self
4038 .capability_evaluator
4039 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
4040 if let Some(conflict) = conflicts.first() {
4041 let message = format!(
4042 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
4043 conflict.capability, conflict.claimants[0], candidate.module_id
4044 );
4045 for conflict in &mut conflicts {
4046 conflict.source = DuplicateClaimSource::CatalogUpdate;
4047 }
4048 log_duplicate_claim_events(conflicts);
4049 return Ok(vec![control_error_frame(
4050 &frame,
4051 "reserved_capability",
4052 message,
4053 )?]);
4054 }
4055
4056 let updated = match self.registry.replace_catalog_for_connection(connection_id, provides, capabilities, ready) {
4057 Err(crate::registry::RegistryError::FederatedCapabilityChanged) => return Ok(vec![control_error_frame(
4058 &frame, error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
4059 "scope-federated/v1 is fixed at HELLO; re-register on a new connection to change it",
4060 )?]),
4061 result => result.map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?,
4062 };
4063 if updated.is_none() {
4064 return Ok(vec![control_error_frame(
4065 &frame,
4066 "not_registered",
4067 "catalog.update requires an active module registration owned by this connection",
4068 )?]);
4069 }
4070 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
4071 log_duplicate_claim_events(
4072 self.capability_evaluator
4073 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
4074 );
4075 }
4076 if capability_census_trigger(
4077 registration.manifest.capabilities.as_ref(),
4078 updated
4079 .as_ref()
4080 .and_then(|entry| entry.manifest.capabilities.as_ref()),
4081 ) {
4082 self.enforce_capability_denies();
4083 }
4084 self.refresh_capability_requirements();
4085
4086 let response = ModuleControlResponseToModule::CatalogUpdate {};
4087 control_response_body_frame(
4088 &frame,
4089 &response,
4090 "ModuleControlResponseToModule::CatalogUpdate",
4091 )
4092 .map(|frame| vec![frame])
4093 }
4094
4095 pub(crate) fn with_activity_ledger_config(
4096 mut self,
4097 run_dir: &Path,
4098 incarnation: String,
4099 config: crate::activity::ActivityConfig,
4100 ) -> Self {
4101 self.activity = Some(crate::activity::ActivityLedger::open_config(
4102 run_dir.join("activity.jsonl"),
4103 incarnation,
4104 config,
4105 ));
4106 self.supervisor.set_activity(self.activity.clone());
4107 self
4108 }
4109
4110 pub(crate) async fn configure_activity(
4111 &self,
4112 config: crate::activity::ActivityConfig,
4113 ) -> Result<(), &'static str> {
4114 match &self.activity {
4115 Some(ledger) => ledger.configure(config).await,
4116 None => Ok(()),
4117 }
4118 }
4119
4120 fn cancel_activity_subscription(
4121 &self,
4122 ctx: &RouteCtx,
4123 frame: &Frame,
4124 ) -> Result<bool, RouterError> {
4125 let task = self
4126 .activity_streams
4127 .lock()
4128 .unwrap_or_else(|p| p.into_inner())
4129 .remove(&(ctx.connection_id, frame.header.corr));
4130 let Some(task) = task else {
4131 return Ok(false);
4132 };
4133 task.abort();
4134 let end = Frame::build_with_version(
4135 response_version(frame),
4136 FrameType::StreamEnd,
4137 control_flags(),
4138 0,
4139 0,
4140 frame.header.corr,
4141 Vec::new(),
4142 )
4143 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
4144 let sink = ctx.egress.clone();
4145 tokio::spawn(async move {
4146 let _ = sink.send(end).await;
4147 });
4148 Ok(true)
4149 }
4150
4151 async fn handle_activity_read(
4152 &self,
4153 ctx: &RouteCtx,
4154 frame: Frame,
4155 ) -> Result<Vec<Frame>, RouterError> {
4156 use subc_control::{
4157 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
4158 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
4159 };
4160 #[derive(Deserialize)]
4161 struct Identity {
4162 #[serde(default)]
4163 consumer_identity: Option<ConsumerIdentity>,
4164 }
4165 let principal = match self
4169 .forwarding
4170 .module_id_for_connection(ctx.connection_id)
4171 .map_err(RouterError::Forwarding)?
4172 {
4173 Some(module_id) => Ok(Principal::Reserved { module_id }),
4174 None => {
4175 let identity: Identity = match serde_json::from_slice(&frame.body) {
4176 Ok(identity) => identity,
4177 Err(_) => {
4178 return Ok(vec![control_error_frame(
4179 &frame,
4180 ACTIVITY_MALFORMED,
4181 "malformed activity read",
4182 )?])
4183 }
4184 };
4185 self.route_open_principal(&frame, identity.consumer_identity)?
4186 }
4187 };
4188 let admitted = matches!(principal, Ok(Principal::Direct))
4189 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
4190 if !admitted {
4191 return Ok(vec![control_error_frame(
4192 &frame,
4193 ACTIVITY_READ_FORBIDDEN,
4194 "activity reads require an operator or trusted mirror principal",
4195 )?]);
4196 }
4197 let Some(ledger) = &self.activity else {
4198 return Ok(vec![control_error_frame(
4199 &frame,
4200 ACTIVITY_UNAVAILABLE,
4201 "activity ledger unavailable",
4202 )?]);
4203 };
4204 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
4205 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
4206 {
4207 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
4208 Ok(request) => ledger.query(request).await,
4209 Err(_) => Err(ACTIVITY_MALFORMED),
4210 };
4211 return match result {
4212 Ok(reply) => Ok(vec![control_response_body_frame(
4213 &frame,
4214 &reply,
4215 ops::ACTIVITY_QUERY,
4216 )?]),
4217 Err(code) => Ok(vec![control_error_frame(
4218 &frame,
4219 code,
4220 "activity query refused",
4221 )?]),
4222 };
4223 }
4224 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
4225 Ok(request) => request,
4226 Err(_) => {
4227 return Ok(vec![control_error_frame(
4228 &frame,
4229 ACTIVITY_MALFORMED,
4230 "malformed activity subscription",
4231 )?])
4232 }
4233 };
4234 let mut subscription = match ledger.subscribe(request.since).await {
4235 Ok(subscription) => subscription,
4236 Err(code) => {
4237 return Ok(vec![control_error_frame(
4238 &frame,
4239 code,
4240 "activity subscription refused",
4241 )?])
4242 }
4243 };
4244 let key = (ctx.connection_id, frame.header.corr);
4245 let sink = ctx.egress.clone();
4246 let mut streams = self
4247 .activity_streams
4248 .lock()
4249 .unwrap_or_else(|p| p.into_inner());
4250 streams.retain(|_, task| !task.is_finished());
4251 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
4252 return Ok(vec![control_error_frame(
4253 &frame,
4254 "duplicate_subscription",
4255 "activity subscription correlation id already in use",
4256 )?]);
4257 }
4258 let task = tokio::spawn(async move {
4259 loop {
4260 let entry = if let Some(entry) = subscription.replay.pop_front() {
4261 Some(entry)
4262 } else {
4263 subscription.receiver.recv().await
4264 };
4265 let Some(entry) = entry else {
4266 break;
4267 };
4268 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
4269 return;
4270 };
4271 let Ok(next) = Frame::build_with_version(
4272 response_version(&frame),
4273 FrameType::StreamData,
4274 control_flags(),
4275 0,
4276 0,
4277 frame.header.corr,
4278 body,
4279 ) else {
4280 return;
4281 };
4282 if sink.send(next).await.is_err() {
4283 return;
4284 }
4285 }
4286 if let Some(code) = subscription.terminal_code() {
4289 if let Ok(end) =
4290 control_error_frame(&frame, code, "activity subscriber fell behind")
4291 {
4292 let _ = sink.send(end).await;
4293 }
4294 }
4295 });
4296 streams.insert(key, task.abort_handle());
4297 Ok(Vec::new())
4298 }
4299
4300 async fn handle_activity_record(
4301 &self,
4302 ctx: &RouteCtx,
4303 frame: Frame,
4304 ) -> Result<Vec<Frame>, RouterError> {
4305 use subc_control::{
4306 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
4307 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
4308 };
4309 let result = async {
4310 let module = self
4311 .forwarding
4312 .module_id_for_connection(ctx.connection_id)
4313 .ok()
4314 .flatten()
4315 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
4316 let request: ActivityRecordRequest =
4317 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
4318 crate::activity::validate(&module, &request)?;
4319 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
4320 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
4321 let at_ms = std::time::SystemTime::now()
4322 .duration_since(std::time::UNIX_EPOCH)
4323 .unwrap_or_default()
4324 .as_millis() as u64;
4325 ledger.record(module, request, actor, at_ms).await
4326 }
4327 .await;
4328 match result {
4329 Ok(entry) => Ok(vec![control_response_body_frame(
4330 &frame,
4331 &ActivityRecordResponse { seq: entry.seq },
4332 "activity.record",
4333 )?]),
4334 Err(code) => Ok(vec![control_error_frame(
4335 &frame,
4336 code,
4337 "activity recording refused",
4338 )?]),
4339 }
4340 }
4341
4342 fn activity_actor(
4343 &self,
4344 connection: ConnectionId,
4345 module: &str,
4346 on: Option<&subc_control::ActivityOn>,
4347 ) -> Result<subc_control::ActivityActor, &'static str> {
4348 use crate::forwarding::{DataRoute, DataRouteState};
4349 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
4350 let principal = |id: &str| Principal::Reserved {
4351 module_id: id.to_owned(),
4352 };
4353 let mut actor = ActivityActor {
4354 principal: principal(module),
4355 scope_owner: None,
4356 agent_id: None,
4357 run_id: None,
4358 flow_id: None,
4359 project_root: None,
4360 workdir: None,
4361 project_id: None,
4362 };
4363 let scope = match on {
4364 None => return Ok(actor),
4365 Some(ActivityOn::Scope(scope)) => {
4366 let selector = subc_protocol::scope::ScopeSelector::new(
4367 principal(module),
4368 scope.scope_ref.clone(),
4369 Some(scope.epoch),
4370 );
4371 self.scopes
4372 .read()
4373 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4374 .attribute(&principal(module), module, &selector, true)
4375 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4376 }
4377 Some(ActivityOn::Route(route)) => {
4378 let DataRoute::Module(DataRouteState::Bound(binding)) = self
4379 .forwarding
4380 .lookup_data_route(connection, route.channel, route.epoch)
4381 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4382 else {
4383 return Err(ACTIVITY_ACTOR_UNKNOWN);
4384 };
4385 if binding.module_id != module {
4386 return Err(ACTIVITY_ACTOR_UNKNOWN);
4387 }
4388 actor.principal = binding.principal.clone();
4389 actor.project_root = binding
4390 .project_root
4391 .as_ref()
4392 .map(|root| root.as_path().to_path_buf());
4393 let Some(bound) = &binding.scope else {
4394 return Ok(actor);
4395 };
4396 let owner = principal(&bound.owner);
4397 let selector = subc_protocol::scope::ScopeSelector::new(
4398 owner.clone(),
4399 bound.scope_ref.clone(),
4400 Some(bound.tag.scope_epoch),
4401 );
4402 let admitted = self
4403 .scopes
4404 .read()
4405 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4406 .attribute(&owner, module, &selector, true)
4407 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4408 if admitted.tag != bound.tag {
4409 return Err(ACTIVITY_ACTOR_UNKNOWN);
4410 }
4411 admitted
4412 }
4413 };
4414 actor.scope_owner = Some(scope.stamp.owner);
4415 actor.agent_id = scope.stamp.attributes.agent_id;
4416 actor.run_id = scope.stamp.attributes.run_id;
4417 actor.flow_id = scope.stamp.attributes.flow_id;
4418 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4419 actor.project_id = scope.stamp.project_id;
4420 Ok(actor)
4421 }
4422
4423 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4424 self.refresh_capability_requirements();
4425 let mut counters = self.counters.snapshot();
4431 let losses = self.resource_tracker.loss_diagnostics();
4432 if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
4433 if let Some(obj) = counters.as_object_mut() {
4434 obj.insert("agent_command_declaration_loss_by_module".into(), losses);
4435 }
4436 }
4437 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4438 obj.insert(
4439 "activity_unparseable_lines".into(),
4440 ledger.read_errors().into(),
4441 );
4442 obj.insert(
4443 "activity_lifecycle_append_failures".into(),
4444 ledger.lifecycle_failures().into(),
4445 );
4446 obj.insert(
4447 "activity_started_digest_skips".into(),
4448 ledger.lifecycle_digest_skips().into(),
4449 );
4450 }
4451 if let (Ok((connections_with_routes, max)), Some(obj)) = (
4452 self.forwarding.client_route_concentration(),
4453 counters.as_object_mut(),
4454 ) {
4455 obj.insert(
4456 "client_connections_with_routes".into(),
4457 connections_with_routes.into(),
4458 );
4459 obj.insert("max_routes_on_one_connection".into(), max.into());
4460 }
4461 if let (Some(open_breakers), Some(obj)) = (
4467 self.route_bind_breakers.open_snapshot(),
4468 counters.as_object_mut(),
4469 ) {
4470 obj.insert("route_bind_breakers_open".into(), open_breakers);
4471 }
4472 if let (Some(reason), Some(obj)) = (
4477 self.local_endpoint_unavailable.clone(),
4478 counters.as_object_mut(),
4479 ) {
4480 obj.insert("local_endpoint_unavailable".into(), reason.into());
4481 }
4482 let response = ClientControlResponse::ServerDescribe {
4483 protocol_ver: PROTOCOL_VERSION,
4484 subc_ops: subc_ops(),
4485 capabilities: self.describe_capabilities(),
4486 connected_clients: self.connected_clients.count(),
4487 counters: Some(counters),
4488 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4489 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4490 capability_requirements: self.capability_requirement_statuses(),
4491 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4492 config_path: self
4493 .rescan
4494 .as_ref()
4495 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4496 };
4497 Ok(vec![control_response_body_frame(
4498 &frame,
4499 &response,
4500 "ClientControlResponse::ServerDescribe",
4501 )?])
4502 }
4503
4504 fn handle_catalog_list(
4505 &self,
4506 frame: Frame,
4507 module_id: Option<String>,
4508 ) -> Result<Vec<Frame>, RouterError> {
4509 let (generation, modules) = self
4510 .registry
4511 .list_modules_with_nonce_verification()
4512 .map_err(|err| {
4513 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4514 })?;
4515 let entries = modules
4516 .into_iter()
4517 .filter(|(registration, _)| {
4518 module_id
4519 .as_deref()
4520 .map(|wanted| registration.manifest.module_id == wanted)
4521 .unwrap_or(true)
4522 })
4523 .map(|(registration, nonce_verified)| {
4524 let not_ready = self.not_ready_reason(®istration);
4525 let roles = registration.manifest.provides;
4526 CatalogEntry::new(
4527 registration.manifest.module_id,
4528 roles,
4529 registration.control_ops,
4530 )
4531 .with_ready(not_ready.is_none())
4532 .with_display_name(registration.manifest.display_name)
4533 .with_not_ready(not_ready)
4534 .with_module_version(Some(registration.manifest.module_version))
4535 .with_provenance(registration.manifest.provenance)
4536 .with_capabilities(registration.manifest.capabilities)
4537 .with_self_signals(registration.manifest.self_signals)
4538 .with_events(registration.manifest.events)
4539 .with_operation_delivery(registration.manifest.operation_delivery)
4540 .with_operation_payload(registration.manifest.operation_payload)
4541 .with_flow_operations(registration.manifest.flow_operations)
4542 .with_confirm_operations(registration.manifest.confirm_operations)
4543 .with_rich_ui(registration.manifest.rich_ui)
4544 .with_role_declarations(registration.manifest.role_declarations)
4545 .with_nonce_verified(nonce_verified)
4546 })
4547 .collect();
4548 let response = ClientControlResponse::CatalogList {
4549 generation,
4550 modules: entries,
4551 subc_ops: subc_ops(),
4552 };
4553 Ok(vec![control_response_body_frame(
4554 &frame,
4555 &response,
4556 "ClientControlResponse::CatalogList",
4557 )?])
4558 }
4559
4560 fn attest_route_open_consumer(
4565 &self,
4566 consumer_identity: Option<&ConsumerIdentity>,
4567 ) -> Option<Principal> {
4568 let Some(consumer_identity) = consumer_identity else {
4569 return Some(Principal::Direct);
4570 };
4571 self.supervisor
4572 .spawned_consumer_authorized(
4573 &consumer_identity.module_id,
4574 &consumer_identity.launch_nonce,
4575 )
4576 .then(|| Principal::Reserved {
4577 module_id: consumer_identity.module_id.clone(),
4578 })
4579 }
4580
4581 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4585 let Ok(ClientControlRequest::RouteOpen {
4586 identity,
4587 consumer_identity,
4588 scope,
4589 ..
4590 }) = parse_client_control_request(&frame.body)
4591 else {
4592 return RouteOpenCaller::unknown();
4593 };
4594 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4595 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4596 }
4597
4598 fn route_open_principal(
4599 &self,
4600 frame: &Frame,
4601 consumer_identity: Option<ConsumerIdentity>,
4602 ) -> Result<Result<Principal, Frame>, RouterError> {
4603 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4604 self.route_open_principal_attested(frame, consumer_identity, attested)
4605 }
4606
4607 fn route_open_principal_attested(
4611 &self,
4612 frame: &Frame,
4613 consumer_identity: Option<ConsumerIdentity>,
4614 attested: Option<Principal>,
4615 ) -> Result<Result<Principal, Frame>, RouterError> {
4616 if let Some(principal) = attested {
4617 return Ok(Ok(principal));
4618 }
4619 let Some(consumer_identity) = consumer_identity else {
4620 return Ok(Ok(Principal::Direct));
4621 };
4622
4623 Ok(Err(control_error_frame(
4624 frame,
4625 "bad_consumer_identity",
4626 format!(
4627 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4628 consumer_identity.module_id
4629 ),
4630 )?))
4631 }
4632
4633 #[allow(clippy::too_many_arguments)]
4640 fn route_open_refusal_frame(
4641 &self,
4642 ctx: &RouteCtx,
4643 caller: &RouteOpenCaller,
4644 frame: &Frame,
4645 module_id: &str,
4646 reason: &'static str,
4647 code: &'static str,
4648 message: impl Into<String>,
4649 ) -> Result<Frame, RouterError> {
4650 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4651 control_error_frame(frame, code, message.into())
4652 }
4653
4654 #[allow(clippy::too_many_arguments)]
4663 fn route_bind_rejection(
4664 &self,
4665 ctx: &RouteCtx,
4666 caller: &RouteOpenCaller,
4667 frame: &Frame,
4668 target_module_id: &str,
4669 body: ErrorBody,
4670 daemon_authored: bool,
4671 ) -> Result<Vec<Frame>, RouterError> {
4672 if daemon_authored {
4678 let scope_code = match body.code.as_str() {
4679 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4680 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4681 _ => None,
4682 };
4683 if let Some(code) = scope_code {
4684 self.observe_route_open_refusal(
4685 ctx,
4686 caller,
4687 target_module_id,
4688 "scope_changed_before_commit",
4689 code,
4690 );
4691 return Ok(vec![control_error_body_frame(frame, body)?]);
4692 }
4693 }
4694 self.counters
4695 .increment_route_open_refused("module_rejected");
4696 info!(
4697 target: "control",
4698 code = "module_rejected",
4699 module_code = ?body.code,
4700 module_id = ?target_module_id,
4701 connection_id = ctx.connection_id.get(),
4702 principal = %caller.principal,
4703 scope = %caller.scope,
4704 harness = %caller.harness,
4705 "route.open refused"
4706 );
4707 let refusal = if daemon_authored {
4708 control_error_body_frame(frame, body)?
4709 } else {
4710 control_relayed_error_body_frame(frame, body)?
4711 };
4712 Ok(vec![refusal])
4713 }
4714
4715 #[allow(clippy::too_many_arguments)]
4733 fn route_open_breaker_refusal_frame(
4734 &self,
4735 ctx: &RouteCtx,
4736 caller: &RouteOpenCaller,
4737 frame: &Frame,
4738 module_id: &str,
4739 consecutive_timeouts: u32,
4740 retry_in: Duration,
4741 probe_in_flight: bool,
4742 ) -> Result<Frame, RouterError> {
4743 self.counters
4744 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4745 debug!(
4746 target: "control",
4747 code = "module_timeout",
4748 module_id = ?module_id,
4749 connection_id = ctx.connection_id.get(),
4750 principal = %caller.principal,
4751 scope = %caller.scope,
4752 harness = %caller.harness,
4753 consecutive_timeouts,
4754 retry_in_ms = retry_in.as_millis() as u64,
4755 probe_in_flight,
4756 "route.open refused by open bind-relay breaker"
4757 );
4758 let detail = if probe_in_flight {
4765 "one test route is already being tried; retry once it settles".to_string()
4766 } else {
4767 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4768 };
4769 control_error_frame(
4770 frame,
4771 "module_timeout",
4772 format!(
4773 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4774 timed out in a row); {detail}; its established routes are unaffected"
4775 ),
4776 )
4777 }
4778
4779 fn observe_route_open_refusal(
4792 &self,
4793 ctx: &RouteCtx,
4794 caller: &RouteOpenCaller,
4795 module_id: &str,
4796 reason: &'static str,
4797 code: &'static str,
4798 ) {
4799 self.counters.increment_route_open_refused(code);
4800 info!(
4801 target: "control",
4802 code,
4803 reason,
4804 module_id = ?module_id,
4805 connection_id = ctx.connection_id.get(),
4806 principal = %caller.principal,
4807 scope = %caller.scope,
4808 harness = %caller.harness,
4809 "route.open refused"
4810 );
4811 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4812 self.route_outages.record_not_serving(module_id, reason);
4813 }
4814 }
4815
4816 fn observe_route_open_accept(
4861 &self,
4862 ctx: &RouteCtx,
4863 module_id: &str,
4864 principal: &str,
4865 scope: &str,
4866 ) {
4867 self.route_outages.record_accepted(module_id);
4868 self.counters.increment_route_open_accepted(principal);
4869 info!(
4870 target: "control",
4871 principal,
4872 module_id,
4873 connection_id = ctx.connection_id.get(),
4874 scope = %scope,
4875 "route.open accepted"
4876 );
4877 }
4878
4879 fn supervised_absent_route_open_refusal_frame(
4880 &self,
4881 ctx: &RouteCtx,
4882 caller: &RouteOpenCaller,
4883 frame: &Frame,
4884 module_id: &str,
4885 code: &'static str,
4886 status: &crate::supervise::ModuleStatus,
4887 ) -> Result<Frame, RouterError> {
4888 self.counters.increment_route_open_refused(code);
4889 info!(
4890 target: "control",
4891 code,
4892 reason = "supervised_not_registered",
4893 module_id = ?module_id,
4894 connection_id = ctx.connection_id.get(),
4895 principal = %caller.principal,
4896 scope = %caller.scope,
4897 harness = %caller.harness,
4898 state = %status.state,
4899 enabled = status.enabled,
4900 live = status.live,
4901 "route.open refused"
4902 );
4903 self.route_outages
4907 .record_not_serving(module_id, "supervised_not_registered");
4908 let message = if let Some(pid) = status.waiting_for_previous_pid {
4909 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4910 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4911 })
4912 } else {
4913 format!(
4914 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4915 status.state, status.enabled, status.live
4916 )
4917 };
4918 control_error_frame(frame, code, message)
4919 }
4920
4921 async fn handle_route_open(
4922 &self,
4923 ctx: &RouteCtx,
4924 frame: Frame,
4925 request: RouteOpenRequest,
4926 timings: &mut RouteOpenTimings,
4927 ) -> Result<Vec<Frame>, RouterError> {
4928 let RouteOpenRequest {
4929 target,
4930 mut identity,
4931 consumer_identity,
4932 consumer_capabilities,
4933 role_versions,
4934 busy_refusals,
4935 admission_facts,
4936 scope,
4937 } = request;
4938 let target_module_id = target_module_id(&target).to_string();
4939 timings.module_id.clone_from(&target_module_id);
4940 if self
4941 .registry
4942 .get_module_by_connection(ctx.connection_id)
4943 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4944 .is_some()
4945 {
4946 return Ok(vec![control_error_frame(
4947 &frame,
4948 "invalid_request",
4949 "module connections cannot open client routes",
4950 )?]);
4951 }
4952 debug!(
4953 connection_id = ctx.connection_id.get(),
4954 corr = frame.header.corr,
4955 module_id = %target_module_id,
4956 "handling route.open"
4957 );
4958
4959 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4964 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4965
4966 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4972 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4973 self.observe_route_open_refusal(
4974 ctx,
4975 &caller,
4976 &target_module_id,
4977 "invalid_role_versions",
4978 error_codes::INVALID_REQUEST,
4979 );
4980 return Ok(vec![control_error_body_frame(
4981 &frame,
4982 ErrorBody {
4983 code: error_codes::INVALID_REQUEST.to_string(),
4984 message: error.to_string(),
4985 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4986 },
4987 )?]);
4988 }
4989
4990 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4991 if status.waiting_for_previous_pid.is_some() {
4992 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4993 ctx,
4994 &caller,
4995 &frame,
4996 &target_module_id,
4997 "module_warming",
4998 &status,
4999 )?]);
5000 }
5001 }
5002
5003 let Some(registration) = self
5022 .registry
5023 .get_module(&target_module_id)
5024 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5025 else {
5026 if let Some((status, warming)) =
5027 self.supervisor_status(&target_module_id, frame.header.corr)?
5028 {
5029 if status.protocol == ModuleProtocol::None {
5036 return Ok(vec![self.route_open_refusal_frame(
5037 ctx,
5038 &caller,
5039 &frame,
5040 &target_module_id,
5041 "protocol_none",
5042 error_codes::MODULE_NO_PROTOCOL,
5043 format!(
5044 "module_id '{target_module_id}' is declared protocol: none; \
5045 it speaks no subc wire and serves no routes"
5046 ),
5047 )?]);
5048 }
5049 let code = if warming {
5050 "module_warming"
5051 } else {
5052 "target_unavailable"
5053 };
5054 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
5055 ctx,
5056 &caller,
5057 &frame,
5058 &target_module_id,
5059 code,
5060 &status,
5061 )?]);
5062 }
5063 if let Some(removed_ago_ms) =
5064 self.supervisor.removal_tombstone_age_ms(&target_module_id)
5065 {
5066 return Ok(vec![self.route_open_refusal_frame(
5067 ctx,
5068 &caller,
5069 &frame,
5070 &target_module_id,
5071 "removed",
5072 error_codes::MODULE_REMOVED,
5073 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
5074 )?]);
5075 }
5076 return Ok(vec![self.route_open_refusal_frame(
5077 ctx,
5078 &caller,
5079 &frame,
5080 &target_module_id,
5081 "not_registered",
5082 error_codes::UNKNOWN_MODULE,
5083 format!("module_id '{target_module_id}' is not registered"),
5084 )?]);
5085 };
5086
5087 if !registration.ready {
5091 self.counters
5092 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
5093 info!(
5094 target: "control",
5095 code = error_codes::MODULE_WARMING,
5096 module_id = ?target_module_id,
5097 connection_id = ctx.connection_id.get(),
5098 principal = %caller.principal,
5099 scope = %caller.scope,
5100 harness = %caller.harness,
5101 reason = "declared_not_ready",
5102 "route.open refused"
5103 );
5104 self.route_outages
5107 .record_not_serving(&target_module_id, "declared_not_ready");
5108 return Ok(vec![control_error_body_frame(
5109 &frame,
5110 ErrorBody {
5111 code: error_codes::MODULE_WARMING.to_string(),
5112 message: format!(
5113 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
5114 ),
5115 detail: Some(serde_json::json!({
5116 "reason": "declared_not_ready"
5117 })),
5118 },
5119 )?]);
5120 }
5121
5122 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
5141 self.counters
5142 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
5143 info!(
5144 target: "control",
5145 code = error_codes::MODULE_WARMING,
5146 module_id = ?target_module_id,
5147 connection_id = ctx.connection_id.get(),
5148 principal = %caller.principal,
5149 scope = %caller.scope,
5150 harness = %caller.harness,
5151 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5152 capability = %capability,
5153 "route.open refused"
5154 );
5155 return Ok(vec![control_error_body_frame(
5156 &frame,
5157 ErrorBody {
5158 code: error_codes::MODULE_WARMING.to_string(),
5159 message: format!(
5160 "module_id '{target_module_id}' requires capability '{capability}', \
5161 which no registered module provides; retry"
5162 ),
5163 detail: Some(serde_json::json!({
5164 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5165 "capability": capability,
5166 })),
5167 },
5168 )?]);
5169 }
5170
5171 if !target_has_required_role(&target, ®istration.manifest.provides) {
5172 return Ok(vec![self.route_open_refusal_frame(
5173 ctx,
5174 &caller,
5175 &frame,
5176 &target_module_id,
5177 "role_not_provided",
5178 "target_unavailable",
5179 format!("module_id '{target_module_id}' does not provide the requested target"),
5180 )?]);
5181 }
5182
5183 if registration.state != ChannelState::Active {
5184 return Ok(vec![self.route_open_refusal_frame(
5185 ctx,
5186 &caller,
5187 &frame,
5188 &target_module_id,
5189 "registration_not_active",
5190 "target_unavailable",
5191 format!("module_id '{target_module_id}' is not active"),
5192 )?]);
5193 }
5194
5195 if self
5196 .forwarding
5197 .module_is_draining(&target_module_id)
5198 .map_err(RouterError::Forwarding)?
5199 {
5200 return Ok(vec![self.route_open_refusal_frame(
5201 ctx,
5202 &caller,
5203 &frame,
5204 &target_module_id,
5205 "reloading",
5206 "module_reloading",
5207 format!("module_id '{target_module_id}' is reloading"),
5208 )?]);
5209 }
5210
5211 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
5212 process_liveness.process_live(&target_module_id) == Some(false)
5213 }) {
5214 if process_liveness.process_replacing(&target_module_id) {
5221 return Ok(vec![self.route_open_refusal_frame(
5222 ctx,
5223 &caller,
5224 &frame,
5225 &target_module_id,
5226 "reloading",
5227 "module_reloading",
5228 format!("module_id '{target_module_id}' is reloading"),
5229 )?]);
5230 }
5231 return Ok(vec![self.route_open_refusal_frame(
5232 ctx,
5233 &caller,
5234 &frame,
5235 &target_module_id,
5236 "supervisor_not_live",
5237 "target_unavailable",
5238 format!("module_id '{target_module_id}' is not live"),
5239 )?]);
5240 }
5241
5242 if !self
5243 .forwarding
5244 .has_live_module_connection(&target_module_id)
5245 .map_err(RouterError::Forwarding)?
5246 {
5247 return Ok(vec![self.route_open_refusal_frame(
5248 ctx,
5249 &caller,
5250 &frame,
5251 &target_module_id,
5252 "no_forwarding_connection",
5253 "target_unavailable",
5254 format!("module_id '{target_module_id}' has no live forwarding connection"),
5255 )?]);
5256 }
5257
5258 if let Some(error) =
5259 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
5260 {
5261 self.observe_route_open_refusal(
5262 ctx,
5263 &caller,
5264 &target_module_id,
5265 "op_not_allowed",
5266 "op_not_allowed",
5267 );
5268 return Ok(vec![error]);
5269 }
5270
5271 let principal =
5272 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
5273 Ok(principal) => principal,
5274 Err(error) => {
5275 self.observe_route_open_refusal(
5276 ctx,
5277 &caller,
5278 &target_module_id,
5279 "bad_consumer_identity",
5280 "bad_consumer_identity",
5281 );
5282 return Ok(vec![error]);
5283 }
5284 };
5285 timings.principal = principal_label(&principal);
5286
5287 let build_class = identity.build_class.take().filter(|build_class| {
5290 if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
5291 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
5292 } else {
5293 if self
5294 .build_class_warned
5295 .lock()
5296 .unwrap_or_else(|p| p.into_inner())
5297 .insert(ctx.connection_id)
5298 {
5299 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
5300 "ignoring build_class from a non-Callosum opener");
5301 }
5302 false
5303 }
5304 });
5305
5306 if let Principal::Reserved {
5310 module_id: opening_module_id,
5311 } = &principal
5312 {
5313 if let Some(opening_registration) = self
5314 .registry
5315 .get_module(opening_module_id)
5316 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5317 {
5318 if let Some(capability) =
5319 denied_capability(&opening_registration.manifest, ®istration.manifest)
5320 {
5321 warn!(
5322 opening_module_id,
5323 target_module_id,
5324 capability,
5325 "refusing route.open because an attested capability deny edge matches"
5326 );
5327 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5328 &frame,
5329 &target_module_id,
5330 "capability_deny_edge",
5331 "capability_forbidden",
5332 format!(
5333 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
5334 ),
5335 )?]);
5336 }
5337 }
5338 }
5339
5340 if admission_facts.is_some() {
5341 let carrier_matches = matches!(
5342 &principal,
5343 Principal::Reserved { module_id }
5344 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
5345 );
5346 if !carrier_matches {
5347 return Ok(vec![self.route_open_refusal_frame(
5348 ctx,
5349 &caller,
5350 &frame,
5351 &target_module_id,
5352 "admission_facts_carrier_not_permitted",
5353 "admission_facts_not_permitted",
5354 "admission facts may only be carried by the configured reserved module",
5355 )?]);
5356 }
5357
5358 let target_allowed = self
5359 .admission_facts_targets
5360 .as_ref()
5361 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
5362 if !target_allowed {
5363 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5364 &frame,
5365 &target_module_id,
5366 "admission_facts_target_not_listed",
5367 "admission_facts_target_not_allowed",
5368 format!(
5369 "admission facts are not permitted for target module_id '{target_module_id}'"
5370 ),
5371 )?]);
5372 }
5373
5374 }
5378
5379 let (bound_scope, mut scope_stamp) = match &scope {
5385 None => (None, None),
5386 Some(selector) => {
5387 let owner_configured = match &selector.owner {
5388 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5394 _ => false,
5395 };
5396 let mut scopes = self
5397 .scopes
5398 .write()
5399 .unwrap_or_else(|poisoned| poisoned.into_inner());
5400 let admitted = scopes
5401 .admit(&principal, &target_module_id, selector, owner_configured)
5402 .and_then(|admission| {
5403 crate::scopes::check_target_flow_support(
5404 &admission.stamp,
5405 &target_module_id,
5406 registration.manifest.capabilities.as_ref(),
5407 )?;
5408 crate::scopes::check_target_agent_run_support(
5409 &admission.stamp,
5410 &target_module_id,
5411 registration.manifest.capabilities.as_ref(),
5412 )?;
5413 let bound = scopes.reserve_bind(
5414 &admission,
5415 &principal,
5416 ctx.connection_id,
5417 selector.hold_presence,
5418 );
5419 Ok((admission, bound))
5420 });
5421 match admitted {
5422 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
5423 Err(refusal) => {
5424 return Ok(vec![self.route_open_refusal_frame(
5425 ctx,
5426 &caller,
5427 &frame,
5428 &target_module_id,
5429 refusal.code,
5430 refusal.code,
5431 refusal.message,
5432 )?]);
5433 }
5434 }
5435 }
5436 };
5437 if let Some(stamp) = &scope_stamp {
5440 caller.scope = route_open_scope_label(stamp);
5441 }
5442
5443 timings.enter(RouteOpenPhase::ProjectRoot);
5451 let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
5452 Ok(project) => project,
5453 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5454 };
5455 timings.enter(RouteOpenPhase::Admission);
5456 let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
5457 Ok(root) => root,
5458 Err(err) => {
5459 return Ok(vec![control_error_frame(
5460 &frame,
5461 "invalid_project_root",
5462 err.to_string(),
5463 )?])
5464 }
5465 };
5466 let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
5472
5473 let _concurrency_guard = match self
5484 .route_bind_concurrency
5485 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5486 {
5487 Ok(guard) => guard,
5488 Err(in_flight) => {
5489 return Ok(vec![self.route_open_target_capacity_refusal(
5490 ctx,
5491 &caller,
5492 &frame,
5493 &target_module_id,
5494 in_flight,
5495 )?]);
5496 }
5497 };
5498
5499 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5502 RouteBindAdmission::Admitted { guard, probe } => {
5503 if probe {
5504 info!(
5505 module_id = %target_module_id,
5506 connection_id = ctx.connection_id.get(),
5507 "route.bind breaker half-open: admitting one probe"
5508 );
5509 }
5510 guard
5511 }
5512 RouteBindAdmission::Refused {
5513 consecutive_timeouts,
5514 retry_in,
5515 probe_in_flight,
5516 } => {
5517 return Ok(vec![self.route_open_breaker_refusal_frame(
5518 ctx,
5519 &caller,
5520 &frame,
5521 &target_module_id,
5522 consecutive_timeouts,
5523 retry_in,
5524 probe_in_flight,
5525 )?]);
5526 }
5527 };
5528
5529 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5534 let relay_deadline = Instant::now() + route_bind_relay_timeout;
5535 let operator_app = self
5536 .operator_apps
5537 .lock()
5538 .unwrap_or_else(|p| p.into_inner())
5539 .get(&ctx.connection_id)
5540 .cloned();
5541 timings.enter(RouteOpenPhase::EgressReservation);
5542 let pending = match self
5543 .forwarding
5544 .begin_route_bind_relay_for(
5545 ctx.connection_id,
5546 ctx.egress.clone(),
5547 response_version(&frame),
5548 frame.header.corr,
5549 &target_module_id,
5550 principal.clone(),
5551 bound_scope,
5552 project_root,
5553 identity.harness.clone(),
5554 build_class,
5555 operator_app.clone(),
5556 relay_deadline,
5557 busy_refusals == Some(true),
5558 )
5559 .await
5560 {
5561 Ok(pending) => pending,
5562 Err(err) => {
5563 return Ok(vec![self.route_open_refusal_frame(
5564 ctx,
5565 &caller,
5566 &frame,
5567 &target_module_id,
5568 "relay_reservation_failed",
5569 forwarding_error_code(&err),
5570 err.to_string(),
5571 )?])
5572 }
5573 };
5574 timings.enter(RouteOpenPhase::Admission);
5575 let crate::forwarding::PendingRouteBindRelay {
5576 endpoint,
5577 module_sink,
5578 negotiated_ver,
5579 client_channel,
5580 client_epoch,
5581 module_channel,
5582 module_epoch,
5583 corr: relay_corr,
5584 receiver,
5585 } = pending;
5586 let mut reservation =
5587 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5588
5589 let relay_registration = self
5594 .registry
5595 .get_module_by_connection(endpoint.connection_id)
5596 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5597 let relay_capabilities = relay_registration
5598 .as_ref()
5599 .and_then(|registration| registration.manifest.capabilities.as_ref());
5600 if let Some(selector) = &scope {
5601 if let Err(refusal) = self
5604 .scopes
5605 .read()
5606 .unwrap_or_else(|poisoned| poisoned.into_inner())
5607 .check_bind_proof(&principal, selector)
5608 {
5609 reservation.release_and_disarm();
5610 return Ok(vec![self.route_open_refusal_frame(
5611 ctx,
5612 &caller,
5613 &frame,
5614 &target_module_id,
5615 refusal.code,
5616 refusal.code,
5617 refusal.message,
5618 )?]);
5619 }
5620 }
5621 if let Some(stamp) = scope_stamp.as_ref() {
5622 if let Err(refusal) = crate::scopes::check_target_flow_support(
5623 stamp,
5624 &target_module_id,
5625 relay_capabilities,
5626 )
5627 .and_then(|()| {
5628 crate::scopes::check_target_agent_run_support(
5629 stamp,
5630 &target_module_id,
5631 relay_capabilities,
5632 )
5633 }) {
5634 reservation.release_and_disarm();
5635 return Ok(vec![self.route_open_refusal_frame(
5636 ctx,
5637 &caller,
5638 &frame,
5639 &target_module_id,
5640 refusal.code,
5641 refusal.code,
5642 refusal.message,
5643 )?]);
5644 }
5645 }
5646
5647 if scope_stamp
5650 .as_ref()
5651 .is_some_and(|stamp| stamp.kind == subc_protocol::scope::ScopeKind::Federated)
5652 && !crate::scopes::target_supports_federated(relay_capabilities)
5653 {
5654 scope_stamp = None;
5656 }
5657 if !relay_capabilities.is_some_and(|caps| {
5658 caps.provides
5659 .iter()
5660 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5661 }) {
5662 if let Some(stamp) = scope_stamp.as_mut() {
5663 stamp.workdir = None;
5664 stamp.project_id = None;
5665 }
5666 }
5667 let worker_run = scope_stamp
5672 .as_ref()
5673 .is_some_and(crate::scopes::is_worker_run)
5674 && relay_capabilities.is_some_and(|caps| {
5675 caps.provides
5676 .iter()
5677 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5678 });
5679 if !worker_run
5680 && !relay_capabilities.is_some_and(|caps| {
5681 caps.provides
5682 .iter()
5683 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5684 })
5685 {
5686 if let Some(stamp) = scope_stamp.as_mut() {
5687 stamp.attributes.on_behalf_of_agent = None;
5688 }
5689 }
5690 if !relay_capabilities.is_some_and(|caps| {
5691 caps.provides
5692 .iter()
5693 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5694 }) {
5695 if let Some(stamp) = scope_stamp.as_mut() {
5696 stamp.attributes.flow_version = None;
5697 stamp.attributes.flow_code_hash = None;
5698 stamp.attributes.flow_install_id = None;
5699 }
5700 }
5701 let call_mode = if worker_run
5702 || relay_capabilities
5703 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5704 {
5705 Some(derive_call_mode(scope_stamp.as_ref()))
5706 } else {
5707 if let Some(stamp) = scope_stamp.as_mut() {
5708 stamp.expires_at_ms = None;
5709 }
5710 None
5711 };
5712
5713 debug!(
5714 connection_id = ctx.connection_id.get(),
5715 client_channel,
5716 client_epoch,
5717 module_channel,
5718 module_epoch,
5719 "reserved route handle pair"
5720 );
5721 let route_principal_label = principal_label(&principal);
5724 let relay = ModuleControlRequest::RouteBind {
5725 route_channel: module_channel,
5726 epoch: module_epoch,
5727 target,
5728 identity,
5729 principal: Some(principal),
5730 consumer_capabilities,
5731 role_versions,
5732 admission_facts,
5733 scope: scope_stamp,
5734 call_mode,
5735 };
5736 let peer = RouteBindPeer::new()
5737 .with_observed_peer(
5738 relay_capabilities
5739 .is_some_and(|caps| {
5740 caps.provides
5741 .iter()
5742 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5743 })
5744 .then(|| {
5745 self.observed_peers
5746 .lock()
5747 .unwrap_or_else(|p| p.into_inner())
5748 .get(&ctx.connection_id)
5749 .cloned()
5750 .unwrap_or_else(ObservedPeer::unobserved)
5751 }),
5752 )
5753 .with_operator_app(
5754 self.connection_operator_app(operator_app.as_ref())
5755 .filter(|_| {
5756 relay_capabilities.is_some_and(|caps| {
5757 caps.provides
5758 .iter()
5759 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5760 })
5761 }),
5762 );
5763 let relay_body = crate::bind_compat::encode_bind(
5764 &relay,
5765 &peer,
5766 &legacy_bind_project,
5767 relay_capabilities,
5768 )
5769 .map_err(|err| {
5770 RouterError::backend(
5771 0,
5772 frame.header.corr,
5773 format!("failed to encode route.bind request: {err}"),
5774 )
5775 })?;
5776 let relay_frame = Frame::build_with_version(
5777 negotiated_ver,
5778 FrameType::Request,
5779 control_flags(),
5780 0,
5781 0,
5782 relay_corr,
5783 relay_body,
5784 )
5785 .map_err(RouterError::FrameBuild)?;
5786
5787 timings.enter(RouteOpenPhase::BindSend);
5788 let sent = module_sink.send(relay_frame).await;
5789 timings.enter(RouteOpenPhase::Admission);
5790 if let Err(err) = sent {
5791 reservation.release_and_disarm();
5792 return Ok(vec![self.route_open_refusal_frame(
5793 ctx,
5794 &caller,
5795 &frame,
5796 &target_module_id,
5797 "relay_send_failed",
5798 "target_unavailable",
5799 err.to_string(),
5800 )?]);
5801 }
5802
5803 #[cfg(test)]
5804 if let Some(hold) = &self.route_bind_relay_sent_hold {
5805 hold.wait().await;
5809 hold.wait().await;
5810 }
5811
5812 if self
5817 .forwarding
5818 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5819 .map_err(RouterError::Forwarding)?
5820 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5821 {
5822 self.send_abandoned_route_bind_goodbye(
5823 &module_sink,
5824 negotiated_ver,
5825 module_channel,
5826 module_epoch,
5827 );
5828 }
5829
5830 timings.enter(RouteOpenPhase::BindAckWait);
5831 let outcome = timeout_at(relay_deadline, receiver).await;
5832 timings.enter(RouteOpenPhase::Admission);
5833 match outcome {
5834 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5835 reservation.disarm();
5836 if breaker.record_accepted() {
5837 info!(
5838 module_id = %target_module_id,
5839 "route.bind breaker closed: the probe was accepted"
5840 );
5841 }
5842 self.observe_route_open_accept(
5843 ctx,
5844 &target_module_id,
5845 &route_principal_label,
5846 &caller.scope,
5847 );
5848 Ok(Vec::new())
5849 }
5850 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5851 reservation.release_and_disarm();
5852 breaker.record_inconclusive();
5856 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5857 }
5858 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5859 reservation.release_and_disarm();
5860 breaker.record_inconclusive();
5861 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5862 }
5863 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5864 reservation.release_and_disarm();
5865 breaker.record_inconclusive();
5866 tracing::warn!(
5873 module_id = %target_module_id,
5874 "route.bind relay abandoned: {message}"
5875 );
5876 Ok(vec![self.route_open_refusal_frame(
5877 ctx,
5878 &caller,
5879 &frame,
5880 &target_module_id,
5881 "relay_abandoned",
5882 "target_unavailable",
5883 message,
5884 )?])
5885 }
5886 Ok(Err(_)) => {
5887 reservation.release_and_disarm();
5888 breaker.record_inconclusive();
5889 Ok(vec![self.route_open_refusal_frame(
5890 ctx,
5891 &caller,
5892 &frame,
5893 &target_module_id,
5894 "relay_waiter_canceled",
5895 "target_unavailable",
5896 "route.bind relay waiter was canceled before the module responded",
5897 )?])
5898 }
5899 Err(_) => {
5900 reservation.release_and_disarm();
5901 if let Some(opened) = breaker.record_timeout(
5905 self.route_bind_breaker_threshold,
5906 self.route_bind_breaker_cooldown,
5907 ) {
5908 warn!(
5909 module_id = %target_module_id,
5910 consecutive_timeouts = opened.consecutive_timeouts,
5911 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5912 reopened_after_probe = opened.reopened_after_probe,
5913 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5914 );
5915 }
5916 tracing::warn!(
5924 module_id = %target_module_id,
5925 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5926 "route.bind relay timed out: module did not ack within budget"
5927 );
5928 Ok(vec![self.route_open_refusal_frame(
5929 ctx,
5930 &caller,
5931 &frame,
5932 &target_module_id,
5933 "relay_timed_out",
5934 "module_timeout",
5935 format!(
5936 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5937 route_bind_relay_timeout
5938 ),
5939 )?])
5940 }
5941 }
5942 }
5943
5944 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5945 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5946 snapshot: self.supervisor.spawn_snapshot(),
5947 };
5948 Ok(vec![control_response_body_frame(
5949 &frame,
5950 &response,
5951 "ClientControlResponse::SupervisorSpawnSnapshot",
5952 )?])
5953 }
5954
5955 fn handle_supervisor_spawn_subscribe(
5956 &self,
5957 ctx: &RouteCtx,
5958 frame: Frame,
5959 since: Option<SpawnCursor>,
5960 ) -> Result<Vec<Frame>, RouterError> {
5961 match self.supervisor.subscribe_spawns(
5962 ctx.connection_id,
5963 frame.header.corr,
5964 response_version(&frame),
5965 since,
5966 ctx.egress.clone(),
5967 ) {
5968 Ok(()) => Ok(Vec::new()),
5969 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5970 Ok(vec![control_error_body_frame(
5971 &frame,
5972 ErrorBody {
5973 code: "spawn_cursor_incarnation_mismatch".to_string(),
5974 message: "spawn cursor belongs to a different daemon incarnation"
5975 .to_string(),
5976 detail: Some(serde_json::json!({
5977 "current_daemon_incarnation": current
5978 })),
5979 },
5980 )?])
5981 }
5982 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5983 &frame,
5984 ErrorBody {
5985 code: "spawn_cursor_too_old".to_string(),
5986 message: "spawn cursor predates the retained event ring".to_string(),
5987 detail: Some(serde_json::json!({
5988 "oldest_retained_cursor": oldest
5989 })),
5990 },
5991 )?]),
5992 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5993 0,
5994 frame.header.corr,
5995 format!("failed to open supervisor spawn subscription: {error}"),
5996 )),
5997 }
5998 }
5999
6000 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6001 let handler = self.clone();
6004 let runtime = tokio::runtime::Handle::current();
6005 let subscriber = tracing::dispatcher::get_default(Clone::clone);
6008 let corr = frame.header.corr;
6009 tokio::task::spawn_blocking(move || {
6010 runtime.block_on(
6011 list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
6012 .with_subscriber(subscriber),
6013 )
6014 })
6015 .await
6016 .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
6017 }
6018
6019 async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6020 let generation = self
6021 .registry
6022 .generation()
6023 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6024 let supervised = self.supervisor.list();
6025 let mut statuses = BTreeMap::new();
6026 let mut not_running = BTreeSet::new();
6027 for module in &supervised {
6028 let status = module.status_for_control("list").map_err(|err| {
6029 RouterError::backend(
6030 0,
6031 frame.header.corr,
6032 format!("failed to read supervisor status: {err}"),
6033 )
6034 })?;
6035 if !status.enabled || !status.process_alive {
6036 not_running.insert(module.module_id().to_owned());
6037 }
6038 statuses.insert(module.module_id().to_owned(), status);
6039 }
6040 let mut process_resources = BTreeMap::new();
6043 for module in &supervised {
6044 process_resources.insert(
6045 module.module_id().to_owned(),
6046 crate::subtree_resources::read_process(module.clone()).await,
6047 );
6048 }
6049 let mut image_readings = BTreeMap::new();
6052 for module in &supervised {
6053 image_readings.insert(
6054 module.module_id().to_owned(),
6055 module.running_image_agreement().await,
6056 );
6057 }
6058 let readings = self.resource_tracker.list(supervised.clone());
6059 let mut modules = Vec::new();
6060 for module in supervised {
6061 let status = statuses
6062 .remove(module.module_id())
6063 .expect("each supervised module has a status snapshot");
6064 let (configured, _) = module.configuration().map_err(|err| {
6065 RouterError::backend(
6066 0,
6067 frame.header.corr,
6068 format!("failed to read module configuration: {err}"),
6069 )
6070 })?;
6071 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
6072 RouterError::backend(
6073 0,
6074 frame.header.corr,
6075 format!("failed to read retry time: {err}"),
6076 )
6077 })?;
6078 let image = image_readings
6079 .remove(module.module_id())
6080 .expect("each supervised module has an image snapshot");
6081 let resources = process_resources.remove(module.module_id());
6082 let mut verdict =
6083 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
6084 verdict.spec = Some(launch_spec_agreement(
6085 &configured,
6086 status.process_alive,
6087 status.launch_spec_digest.as_ref(),
6088 ));
6089 let pending_reload = Some(verdict);
6090 let configured_enabled = self
6091 .capability_evaluator
6092 .configured_enabled(&status.module_id);
6093 modules.push(
6096 SupervisorEntry::new(
6097 status.module_id,
6098 status.state.to_string(),
6099 status.enabled,
6100 status.live,
6101 status.health.status,
6102 )
6103 .with_configured_enabled(configured_enabled)
6104 .with_next_retry_at_ms(next_retry_at_ms)
6105 .with_launch_nonce_env(Some(!cfg!(unix)))
6106 .with_launch_nonce_source(status.launch_nonce_source)
6107 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
6108 .with_protocol(status.protocol)
6109 .with_pending_reload(pending_reload)
6110 .with_last_probe_ms(status.health.last_probe_ms)
6111 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
6112 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
6113 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
6114 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
6115 .with_restart_count(Some(status.restart_count))
6116 .with_max_restarts(Some(status.max_restarts))
6117 .with_lifetime_restarts(Some(status.lifetime_restarts))
6118 .with_spawn_generation(Some(status.spawn_generation))
6119 .with_restart_window_secs(Some(status.restart_window.as_secs()))
6120 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
6121 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
6122 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
6123 .with_resources(resources),
6124 );
6125 }
6126 for module in &mut modules {
6127 if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
6128 module.tree = tree.clone();
6129 module.cpu_last_hour = hour.clone();
6130 }
6131 module.tree_unavailable = readings
6132 .reasons
6133 .get(&module.module_id)
6134 .map(|reason| (*reason).to_owned());
6135 if module.tree.is_none() {
6136 module.tree_unavailable = Some(
6137 if not_running.contains(&module.module_id) {
6138 "not_running"
6139 } else if let Some(reason) = readings.reasons.get(&module.module_id) {
6140 reason
6141 } else {
6142 "not_yet_measured"
6143 }
6144 .to_owned(),
6145 );
6146 }
6147 }
6148 let response = ClientControlResponse::SupervisorList {
6149 generation,
6150 modules,
6151 daemon: readings.daemon,
6152 agent_commands: readings.agent_commands,
6153 };
6154 Ok(vec![control_response_body_frame(
6155 &frame,
6156 &response,
6157 "ClientControlResponse::SupervisorList",
6158 )?])
6159 }
6160
6161 fn handle_supervisor_stderr_tail(
6162 &self,
6163 frame: Frame,
6164 module_id: String,
6165 max_lines: Option<u32>,
6166 max_bytes: Option<u32>,
6167 ) -> Result<Vec<Frame>, RouterError> {
6168 let Some(module) = self.supervisor.get(&module_id) else {
6169 return Ok(vec![control_error_frame(
6170 &frame,
6171 "unknown_module",
6172 format!("module_id '{module_id}' is not supervised"),
6173 )?]);
6174 };
6175
6176 let snapshot = module.stderr_tail(
6177 max_lines.map(|value| value as usize),
6178 max_bytes.map(|value| value as usize),
6179 );
6180
6181 let response = ClientControlResponse::SupervisorStderrTail {
6182 module_id,
6183 tail: StderrTail {
6184 capture: match snapshot.capture {
6185 CaptureState::Captured => StderrCaptureState::Captured,
6186 CaptureState::Incomplete { reason } => {
6187 StderrCaptureState::Incomplete { reason }
6188 }
6189 CaptureState::NotCaptured { reason } => {
6190 StderrCaptureState::NotCaptured { reason }
6191 }
6192 },
6193 entries: snapshot
6194 .entries
6195 .into_iter()
6196 .map(|entry| match entry {
6197 TailEntry::Line {
6198 text,
6199 truncated,
6200 at_ms,
6201 } => StderrTailEntry::Line {
6202 text,
6203 truncated,
6204 at_ms,
6205 },
6206 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
6207 })
6208 .collect(),
6209 dropped_lines: snapshot.dropped_lines,
6210 },
6211 };
6212 Ok(vec![control_response_body_frame(
6213 &frame,
6214 &response,
6215 "ClientControlResponse::SupervisorStderrTail",
6216 )?])
6217 }
6218
6219 async fn observe_control_wait<F: std::future::Future>(
6220 &self,
6221 ctx: &RouteCtx,
6222 target: Option<&str>,
6223 kind: &'static str,
6224 future: F,
6225 ) -> F::Output {
6226 self.counters
6227 .observe_reader_wait(
6228 || {
6229 let module = self
6230 .forwarding
6231 .module_id_for_connection(ctx.connection_id)
6232 .ok()
6233 .flatten();
6234 let principal = if module.is_some() { "module" } else { "direct" };
6235 (
6236 self.resolve_reader_wait_target(kind, target)
6237 .or(module)
6238 .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
6239 principal.into(),
6240 )
6241 },
6242 kind,
6243 future,
6244 )
6245 .await
6246 }
6247
6248 pub(crate) fn resolve_reader_wait_target(
6251 &self,
6252 op: &str,
6253 selector: Option<&str>,
6254 ) -> Option<String> {
6255 let selector = selector?;
6256 match op {
6257 ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
6258 .supervisor
6259 .get(selector)
6260 .map(|module| module.module_id().to_owned()),
6261 ops::SUPERVISOR_HEALTH_PROBE => self
6262 .registry
6263 .get_module(selector)
6264 .ok()
6265 .flatten()
6266 .map(|registration| registration.manifest.module_id),
6267 _ => None,
6268 }
6269 }
6270
6271 async fn handle_supervisor_terminals(
6272 &self,
6273 ctx: &RouteCtx,
6274 frame: Frame,
6275 module_id: String,
6276 ) -> Result<Vec<Frame>, RouterError> {
6277 let Some(module) = self.supervisor.get(&module_id) else {
6278 return Ok(vec![control_error_frame(
6279 &frame,
6280 "unknown_module",
6281 format!("module_id '{module_id}' is not supervised"),
6282 )?]);
6283 };
6284
6285 let terminals = self
6288 .observe_control_wait(
6289 ctx,
6290 Some(module.module_id()),
6291 ops::SUPERVISOR_TERMINALS,
6292 module.read_durable_terminal_history(),
6293 )
6294 .await
6295 .map_err(|error| {
6296 RouterError::backend(
6297 0,
6298 frame.header.corr,
6299 format!("failed to read terminal history: {error}"),
6300 )
6301 })?;
6302 let response = ClientControlResponse::SupervisorTerminals {
6303 module_id,
6304 terminals,
6305 };
6306 Ok(vec![control_response_body_frame(
6307 &frame,
6308 &response,
6309 "ClientControlResponse::SupervisorTerminals",
6310 )?])
6311 }
6312
6313 fn handle_supervisor_routes(
6314 &self,
6315 frame: Frame,
6316 module_id: Option<String>,
6317 ) -> Result<Vec<Frame>, RouterError> {
6318 let modules = self
6319 .forwarding
6320 .route_census(module_id.as_deref())
6321 .map_err(RouterError::Forwarding)?
6322 .into_iter()
6323 .map(|(module_id, routes)| SupervisorRouteModule {
6324 module_id,
6325 routes: routes
6326 .into_iter()
6327 .map(|route| SupervisorRoute {
6328 consumer: match route.principal {
6329 Principal::Reserved { module_id } => {
6330 SupervisorRouteConsumer::Reserved { module_id }
6331 }
6332 Principal::Direct | Principal::Unverified => {
6333 SupervisorRouteConsumer::Direct {
6334 connection_id: route.goodbye_target.connection_id.get(),
6335 }
6336 }
6337 },
6338 age_ms: Instant::now()
6339 .saturating_duration_since(route.bound_at)
6340 .as_millis()
6341 .try_into()
6342 .unwrap_or(u64::MAX),
6343 draining: route.draining,
6344 drain_reason: route.drain_reason,
6345 })
6346 .collect(),
6347 })
6348 .collect();
6349 let response = ClientControlResponse::SupervisorRoutes { modules };
6350 Ok(vec![control_response_body_frame(
6351 &frame,
6352 &response,
6353 "ClientControlResponse::SupervisorRoutes",
6354 )?])
6355 }
6356
6357 async fn handle_supervisor_provenance(
6358 &self,
6359 ctx: &RouteCtx,
6360 frame: Frame,
6361 module_id: Option<String>,
6362 ) -> Result<Vec<Frame>, RouterError> {
6363 let targeted = module_id.is_some();
6364 let selected = if let Some(module_id) = module_id {
6365 let Some(module) = self.supervisor.get(&module_id) else {
6366 return Ok(vec![control_error_frame(
6367 &frame,
6368 "unknown_module",
6369 format!("module_id '{module_id}' is not supervised"),
6370 )?]);
6371 };
6372 vec![module]
6373 } else {
6374 self.supervisor.list()
6375 };
6376
6377 let target = targeted.then(|| selected[0].module_id());
6378 self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
6379 let mut modules = Vec::with_capacity(selected.len());
6380 for module in &selected {
6381 let (status, observed_image) = module
6382 .status_and_running_image_agreement()
6383 .await
6384 .map_err(|err| {
6385 RouterError::backend(
6386 0,
6387 frame.header.corr,
6388 format!("failed to read supervisor status: {err}"),
6389 )
6390 })?;
6391 let module_declared = self
6392 .registry
6393 .get_module(&status.module_id)
6394 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6395 .and_then(|registration| registration.manifest.provenance)
6396 .map(|build| ModuleDeclaredProvenance::Reported { build })
6397 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
6398 #[cfg(test)]
6399 let running_image = match &self.provenance_probe_override {
6400 Some(result) => result.clone(),
6401 None => observed_image,
6402 };
6403 #[cfg(not(test))]
6404 let running_image = observed_image;
6405 modules.push(SupervisorModuleProvenance {
6406 module_id: status.module_id,
6407 module_declared,
6408 daemon_observed: SupervisorObservedProcess {
6409 launch_nonce_source: status.launch_nonce_source,
6410 pid: status.pid,
6411 spawned_at_ms: status.spawned_at_ms,
6412 spawned_from: status.spawned_from,
6413 running_image,
6414 },
6415 });
6416 }
6417 let daemon = SupervisorDaemonProvenance {
6418 daemon_build: self.daemon_provenance.build.clone(),
6419 daemon_observed: DaemonObservedProcess {
6420 pid: self.daemon_provenance.pid,
6421 started_at_ms: self
6422 .daemon_provenance
6423 .start_clock
6424 .map(|clock| clock.started_at_ms())
6425 .or(self.daemon_provenance.started_at_ms),
6426 running_image: self
6427 .daemon_provenance
6428 .probe
6429 .observe(
6430 self.daemon_provenance.pid,
6431 self.daemon_provenance.executable_path.as_deref(),
6432 self.daemon_provenance.executable_identity.clone(),
6433 self.daemon_provenance.process_start_time,
6434 )
6435 .await,
6436 },
6437 };
6438 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
6439 Ok(vec![control_response_body_frame(
6440 &frame,
6441 &response,
6442 "ClientControlResponse::SupervisorProvenance",
6443 )?])
6444 })
6445 .await
6446 }
6447
6448 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6449 self.refresh_capability_requirements();
6450 let generation = self
6451 .registry
6452 .generation()
6453 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6454 let modules = self
6455 .supervisor
6456 .list()
6457 .into_iter()
6458 .map(|module| {
6459 let status = module.status_for_control("health").map_err(|err| {
6460 RouterError::backend(
6461 0,
6462 frame.header.corr,
6463 format!("failed to read supervisor health: {err}"),
6464 )
6465 })?;
6466 let module_id = status.module_id;
6467 let capability_detail = self
6468 .capability_evaluator
6469 .required_problem_detail(&module_id);
6470 Ok(SupervisorHealthEntry {
6471 module_id,
6472 status: status.health.status,
6473 detail: append_capability_problem_detail(
6474 status.health.detail,
6475 capability_detail,
6476 ),
6477 metrics: status.health.metrics,
6478 consecutive_failures: status.health.consecutive_failures,
6479 late_answer_count: status.health.late_answer_count,
6480 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
6481 last_action: status.health.last_action,
6482 last_action_ms: status.health.last_action_ms,
6483 last_probe_ms: status.health.last_probe_ms,
6484 })
6485 })
6486 .collect::<Result<Vec<_>, RouterError>>()?;
6487 let response = ClientControlResponse::SupervisorHealth {
6488 generation,
6489 modules,
6490 };
6491 Ok(vec![control_response_body_frame(
6492 &frame,
6493 &response,
6494 "ClientControlResponse::SupervisorHealth",
6495 )?])
6496 }
6497
6498 async fn handle_supervisor_restart(
6499 &self,
6500 frame: Frame,
6501 module_id: String,
6502 drain_timeout_ms: Option<u64>,
6503 requester: Principal,
6504 ) -> Result<Vec<Frame>, RouterError> {
6505 let operation_lock = self.supervisor.operation_lock();
6506 let _operation_guard = operation_lock.lock().await;
6507 let Some(module) = self.supervisor.get(&module_id) else {
6508 return Ok(vec![control_error_frame(
6509 &frame,
6510 "unknown_module",
6511 format!("module_id '{module_id}' is not supervised"),
6512 )?]);
6513 };
6514
6515 self.route_outages.mark_operator_action(&module_id);
6516 if let Err(err) = module
6517 .restart_requested(drain_timeout_ms, Some(requester))
6518 .await
6519 {
6520 self.route_outages
6521 .operator_action_ended_unrefused(&module_id);
6522 let (code, message) = match err {
6523 crate::supervise::SuperviseError::Disabled { .. } => {
6524 ("module_disabled", err.to_string())
6525 }
6526 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6527 ("swap_in_progress", err.to_string())
6528 }
6529 _ => (
6530 "target_unavailable",
6531 format!("failed to restart module_id '{module_id}': {err}"),
6532 ),
6533 };
6534 return Ok(vec![control_error_frame(&frame, code, message)?]);
6535 }
6536
6537 let response = ClientControlResponse::SupervisorAck {
6538 module_id,
6539 applied: true,
6540 };
6541 Ok(vec![control_response_body_frame(
6542 &frame,
6543 &response,
6544 "ClientControlResponse::SupervisorAck",
6545 )?])
6546 }
6547
6548 async fn handle_supervisor_swap(
6552 &self,
6553 frame: Frame,
6554 module_id: String,
6555 ready_timeout_ms: Option<u64>,
6556 requester: Principal,
6557 ) -> Result<Vec<Frame>, RouterError> {
6558 let module = {
6565 let operation_lock = self.supervisor.operation_lock();
6566 let _operation_guard = operation_lock.lock().await;
6567 self.supervisor.get(&module_id)
6568 };
6569 let Some(module) = module else {
6570 return Ok(vec![control_error_frame(
6571 &frame,
6572 "unknown_module",
6573 format!("module_id '{module_id}' is not supervised"),
6574 )?]);
6575 };
6576
6577 self.route_outages.mark_operator_action(&module_id);
6578 if let Err(err) = module
6579 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6580 .await
6581 {
6582 self.route_outages
6583 .operator_action_ended_unrefused(&module_id);
6584 use crate::supervise::SuperviseError;
6585 let message = err.to_string();
6586 let error = match err {
6587 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6588 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6589 code: "swap_refused".to_string(),
6590 message,
6591 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6592 },
6593 SuperviseError::SwapFailed {
6594 arm,
6595 candidate_exit,
6596 ..
6597 } => ErrorBody {
6598 code: "swap_failed".to_string(),
6599 message,
6600 detail: Some(serde_json::json!({
6601 "arm": arm.as_str(),
6602 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6603 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6604 })),
6605 },
6606 _ => ErrorBody::new(
6607 "target_unavailable",
6608 format!("failed to swap module_id '{module_id}': {message}"),
6609 ),
6610 };
6611 return Ok(vec![control_error_body_frame(&frame, error)?]);
6612 }
6613 self.route_outages
6617 .operator_action_ended_unrefused(&module_id);
6618
6619 let response = ClientControlResponse::SupervisorAck {
6620 module_id,
6621 applied: true,
6622 };
6623 Ok(vec![control_response_body_frame(
6624 &frame,
6625 &response,
6626 "ClientControlResponse::SupervisorAck",
6627 )?])
6628 }
6629
6630 async fn handle_supervisor_reload(
6631 &self,
6632 frame: Frame,
6633 module_id: String,
6634 requester: Principal,
6635 ) -> Result<Vec<Frame>, RouterError> {
6636 let operation_lock = self.supervisor.operation_lock();
6637 let _operation_guard = operation_lock.lock().await;
6638 let Some(module) = self.supervisor.get(&module_id) else {
6639 return Ok(vec![control_error_frame(
6640 &frame,
6641 "unknown_module",
6642 format!("module_id '{module_id}' is not supervised"),
6643 )?]);
6644 };
6645
6646 self.route_outages.mark_operator_action(&module_id);
6647 if let Err(err) = module.reload_requested(Some(requester)).await {
6648 self.route_outages
6649 .operator_action_ended_unrefused(&module_id);
6650 let (code, message) = match err {
6651 crate::supervise::SuperviseError::Disabled { .. } => {
6652 ("module_disabled", err.to_string())
6653 }
6654 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6655 ("swap_in_progress", err.to_string())
6656 }
6657 _ => (
6658 "reload_failed",
6659 format!("failed to reload module_id '{module_id}': {err}"),
6660 ),
6661 };
6662 return Ok(vec![control_error_frame(&frame, code, message)?]);
6663 }
6664
6665 let response = ClientControlResponse::SupervisorAck {
6666 module_id,
6667 applied: true,
6668 };
6669 Ok(vec![control_response_body_frame(
6670 &frame,
6671 &response,
6672 "ClientControlResponse::SupervisorAck",
6673 )?])
6674 }
6675
6676 async fn handle_supervisor_rescan(
6680 &self,
6681 frame: Frame,
6682 mode: RescanMode,
6683 requester: Principal,
6684 ) -> Result<Vec<Frame>, RouterError> {
6685 let preview = mode.previews();
6686 let Some(context) = self.rescan.clone() else {
6687 return Ok(vec![control_error_frame(
6688 &frame,
6689 "rescan_unavailable",
6690 "the daemon was not started with a reloadable config path".to_string(),
6691 )?]);
6692 };
6693
6694 let operation_lock = self.supervisor.operation_lock();
6695 let _operation_guard = operation_lock.lock().await;
6696 let loaded = match self.load_rescan_config(&mode, &context) {
6700 Ok(config) => config,
6701 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6702 };
6703 let Some((config, activity_config)) = loaded else {
6711 return Ok(vec![control_error_frame(
6712 &frame,
6713 "invalid_daemon_config",
6714 format!(
6715 "daemon config not found at {}; refusing to rescan (an absent config would \
6716 retire every supervised module)",
6717 context.config_path.display()
6718 ),
6719 )?]);
6720 };
6721 let (
6722 configured_port,
6723 storage_config,
6724 admission_facts_carrier_module_id,
6725 admission_facts_targets,
6726 scope_authority_owners,
6727 federated_scope_owner,
6728 privileged_operator_prompt,
6729 modules,
6730 reserved_capabilities,
6731 ) = (
6732 config.port,
6733 config.storage,
6734 config.admission_facts_carrier_module_id,
6735 config.admission_facts_targets,
6736 config.scope_authority_owners,
6737 config.federated_scope_owner,
6738 config.privileged_operator_prompt,
6739 config.modules,
6740 config.reserved_capabilities,
6741 );
6742
6743 let mut restart_required = Vec::new();
6752 for section in RestartRequiredSection::ALL {
6753 let changed = match section {
6754 RestartRequiredSection::Port => configured_port != context.configured_port,
6755 RestartRequiredSection::Storage => storage_config != context.storage_config,
6756 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6757 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6758 }
6759 RestartRequiredSection::AdmissionFactsTargets => {
6760 admission_facts_targets != context.admission_facts_targets
6761 }
6762 RestartRequiredSection::ScopeAuthorityOwners => {
6763 scope_authority_owners != context.scope_authority_owners
6764 }
6765 RestartRequiredSection::FederatedScopeOwner => {
6766 federated_scope_owner != context.federated_scope_owner
6767 }
6768 RestartRequiredSection::PrivilegedOperatorPrompt => {
6769 privileged_operator_prompt != context.privileged_operator_prompt
6770 }
6771 };
6772 if changed {
6773 restart_required.push(section.label().to_string());
6774 }
6775 }
6776 if !restart_required.is_empty() {
6777 warn!(
6778 config_path = %context.config_path.display(),
6779 sections = %restart_required.join(", "),
6780 "daemon config changed outside the modules section; restart the daemon to apply those changes"
6781 );
6782 }
6783
6784 for configured in &modules {
6785 if let Err(err) = validate_spec(&configured.module_spec()) {
6786 return Ok(vec![control_error_frame(
6787 &frame,
6788 "invalid_daemon_config",
6789 format!("supervisor rescan rejected daemon config: {err}"),
6790 )?]);
6791 }
6792 }
6793
6794 let configured_capabilities = modules
6795 .iter()
6796 .map(|module| (module.module_id.clone(), module.enabled))
6797 .collect::<Vec<_>>();
6798 let preview_capability_warnings = if preview {
6799 let (_, registrations) = self.runtime_capability_snapshot()?;
6800 let current_modules = self
6801 .supervisor
6802 .list()
6803 .into_iter()
6804 .map(|module| module.module_id().to_string())
6805 .collect::<BTreeSet<_>>();
6806 let resulting_modules = configured_capabilities.clone();
6807 let removed = current_modules
6808 .into_iter()
6809 .filter(|module_id| {
6810 !resulting_modules
6811 .iter()
6812 .any(|(configured_id, _)| configured_id == module_id)
6813 })
6814 .collect::<Vec<_>>();
6815 let disabled = match &mode {
6818 RescanMode::Candidate(_) => resulting_modules
6819 .iter()
6820 .filter(|(_, enabled)| !enabled)
6821 .map(|(module_id, _)| module_id.clone())
6822 .collect::<Vec<_>>(),
6823 _ => Vec::new(),
6824 };
6825 self.capability_evaluator.preview_warnings(
6826 resulting_modules,
6827 &removed,
6828 &disabled,
6829 ®istrations,
6830 )
6831 } else {
6832 Vec::new()
6833 };
6834 let reconciled = match &mode {
6835 RescanMode::Candidate(candidate) => self
6836 .preview_candidate(&context, modules, candidate)
6837 .map(|result| (result, Vec::new())),
6838 _ => {
6839 self.reconcile_supervised_modules(
6840 &context.supervisor,
6841 modules,
6842 preview,
6843 requester.clone(),
6844 )
6845 .await
6846 }
6847 };
6848 let (result, changed) = match reconciled {
6849 Ok(result) => result,
6850 Err(message) => {
6851 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6852 }
6853 };
6854 if !preview {
6855 if let Some(ledger) = &self.activity {
6856 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6857 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6858 }
6859 if let Err(code) = self.configure_activity(activity_config).await {
6860 warn!(
6861 code,
6862 "activity retention could not be applied during rescan"
6863 );
6864 }
6865 self.capability_evaluator
6866 .configure(configured_capabilities, reserved_capabilities);
6867 self.capability_evaluator.wake_deadline_loop();
6868 self.refresh_capability_requirements();
6869 }
6870 let mut result = result;
6871 result.restart_required = restart_required;
6872 result.capability_warnings = preview_capability_warnings;
6873 let response = ClientControlResponse::SupervisorRescan { result };
6874 Ok(vec![control_response_body_frame(
6875 &frame,
6876 &response,
6877 "ClientControlResponse::SupervisorRescan",
6878 )?])
6879 }
6880
6881 async fn handle_supervisor_release_reserved(
6882 &self,
6883 frame: Frame,
6884 module_id: String,
6885 ) -> Result<Vec<Frame>, RouterError> {
6886 let Some(context) = self.rescan.clone() else {
6887 return Ok(vec![control_error_frame(
6888 &frame,
6889 "release_unavailable",
6890 "reserved-id release requires a daemon started with a reloadable config path",
6891 )?]);
6892 };
6893 let operation_lock = self.supervisor.operation_lock();
6894 let _operation_guard = operation_lock.lock().await;
6895 let loaded = match crate::daemon_config::load(&context.config_path) {
6896 Ok(Some(config)) => config,
6897 Ok(None) => {
6898 return Ok(vec![control_error_frame(
6899 &frame,
6900 "invalid_daemon_config",
6901 format!(
6902 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6903 context.config_path.display()
6904 ),
6905 )?])
6906 }
6907 Err(err) => {
6908 return Ok(vec![control_error_frame(
6909 &frame,
6910 "invalid_daemon_config",
6911 format!("unable to verify reserved-id release against daemon config: {err}"),
6912 )?])
6913 }
6914 };
6915 if loaded
6916 .modules
6917 .iter()
6918 .any(|configured| configured.module_id == module_id)
6919 {
6920 return Ok(vec![control_error_frame(
6921 &frame,
6922 "reserved_module_configured",
6923 format!(
6924 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6925 ),
6926 )?]);
6927 }
6928 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6929 return Ok(vec![control_error_frame(
6930 &frame,
6931 "reserved_gate_not_retained",
6932 format!(
6933 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6934 ),
6935 )?]);
6936 }
6937
6938 let response = ClientControlResponse::SupervisorAck {
6939 module_id,
6940 applied: true,
6941 };
6942 Ok(vec![control_response_body_frame(
6943 &frame,
6944 &response,
6945 "ClientControlResponse::SupervisorAck",
6946 )?])
6947 }
6948
6949 fn plan_rescan(
6953 &self,
6954 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6955 ) -> Result<RescanPlan, String> {
6956 let mut current = BTreeMap::new();
6957 for module in self.supervisor.list() {
6958 let (spec, health) = module.configuration().map_err(|err| {
6959 format!(
6960 "failed to read configuration for module_id '{}': {err}",
6961 module.module_id()
6962 )
6963 })?;
6964 let enabled = module
6965 .status()
6966 .map_err(|err| {
6967 format!(
6968 "failed to read status for module_id '{}': {err}",
6969 module.module_id()
6970 )
6971 })?
6972 .enabled;
6973 current.insert(
6974 module.module_id().to_string(),
6975 (module, spec, health, enabled),
6976 );
6977 }
6978 let configured = configured_modules
6979 .into_iter()
6980 .map(|module| (module.module_id.clone(), module))
6981 .collect::<BTreeMap<_, _>>();
6982
6983 let added = configured
6984 .keys()
6985 .filter(|module_id| !current.contains_key(*module_id))
6986 .cloned()
6987 .collect::<Vec<_>>();
6988 let removed = current
6989 .keys()
6990 .filter(|module_id| !configured.contains_key(*module_id))
6991 .cloned()
6992 .collect::<Vec<_>>();
6993 let mut changed_pending_reload = Vec::new();
6994 let mut configuration_changes = BTreeSet::new();
6995 let mut enabled_changes = BTreeSet::new();
6996 let mut unchanged = 0_u32;
6997
6998 for (module_id, configured_module) in &configured {
6999 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
7000 else {
7001 continue;
7002 };
7003 let launch_changed = *current_spec != configured_module.module_spec();
7007 let configuration_changed =
7008 launch_changed || *current_health != configured_module.health;
7009 let enabled_changed = *current_enabled != configured_module.enabled
7012 || self
7013 .capability_evaluator
7014 .configured_enabled(module_id)
7015 .is_some_and(|enabled| enabled != configured_module.enabled);
7016 if configuration_changed {
7017 configuration_changes.insert(module_id.clone());
7018 }
7019 if launch_changed {
7020 changed_pending_reload.push(module_id.clone());
7021 }
7022 if enabled_changed {
7023 enabled_changes.insert(module_id.clone());
7024 }
7025 if !configuration_changed && !enabled_changed {
7026 unchanged = unchanged.saturating_add(1);
7027 }
7028 }
7029 Ok(RescanPlan {
7030 current,
7031 configured,
7032 added,
7033 removed,
7034 changed_pending_reload,
7035 configuration_changes,
7036 enabled_changes,
7037 unchanged,
7038 })
7039 }
7040
7041 async fn reconcile_supervised_modules(
7050 &self,
7051 supervisor: &Supervisor,
7052 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
7053 preview: bool,
7054 requester: Principal,
7055 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
7056 let plan = self.plan_rescan(configured_modules)?;
7057
7058 if preview {
7064 return Ok((plan.preview_result(), Vec::new()));
7065 }
7066 let changed = plan
7067 .configuration_changes
7068 .union(&plan.enabled_changes)
7069 .cloned()
7070 .collect::<Vec<_>>();
7071 let RescanPlan {
7072 current,
7073 configured,
7074 added,
7075 removed,
7076 changed_pending_reload,
7077 configuration_changes,
7078 enabled_changes,
7079 unchanged,
7080 } = plan;
7081 if !(added.is_empty()
7086 && removed.is_empty()
7087 && configuration_changes.is_empty()
7088 && enabled_changes.is_empty())
7089 {
7090 if let Some(context) = &self.rescan {
7091 context
7092 .generation
7093 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
7094 }
7095 }
7096
7097 for module_id in &removed {
7098 let module = ¤t
7099 .get(module_id)
7100 .expect("removed module came from current supervisor state")
7101 .0;
7102 module
7103 .retire_requested(Some(requester.clone()))
7104 .await
7105 .map_err(|err| {
7106 format!("failed to retire module_id '{module_id}' during rescan: {err}")
7107 })?;
7108 self.supervisor.record_rescan_removal(module_id);
7142 self.supervisor.retire(module_id);
7143 self.route_outages.forget(module_id);
7144 }
7145
7146 for module_id in configured.keys() {
7147 let Some((module, _, _, current_enabled)) = current.get(module_id) else {
7148 continue;
7149 };
7150 let configured_module = configured
7151 .get(module_id)
7152 .expect("configured module id came from configured map");
7153 if configuration_changes.contains(module_id) {
7154 module
7155 .update_configuration(
7156 configured_module.module_spec(),
7157 configured_module.health.clone(),
7158 configured_module.drain_timeout_ms,
7159 )
7160 .await
7161 .map_err(|err| {
7162 format!(
7163 "failed to update module_id '{module_id}' configuration during rescan: {err}"
7164 )
7165 })?;
7166 }
7167 if *current_enabled != configured_module.enabled {
7170 self.route_outages.mark_operator_action(module_id);
7173 module
7174 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
7175 .await
7176 .map_err(|err| {
7177 self.route_outages.operator_action_ended_unrefused(module_id);
7178 format!(
7179 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
7180 configured_module.enabled
7181 )
7182 })?;
7183 }
7184 }
7185
7186 for module_id in &added {
7187 let configured_module = configured
7188 .get(module_id)
7189 .expect("added module id came from configured map");
7190 supervisor
7191 .supervise_configured_requested(
7192 configured_module.module_spec(),
7193 configured_module.enabled,
7194 configured_module.health.clone(),
7195 configured_module.drain_timeout_ms,
7196 configured_module.restart,
7197 configured_module.register_timeout,
7198 Some(requester.clone()),
7199 )
7200 .map_err(|err| {
7201 format!("failed to add module_id '{module_id}' during rescan: {err}")
7202 })?;
7203 }
7204
7205 Ok((
7206 SupervisorRescanResult {
7207 added,
7208 removed,
7209 changed_pending_reload,
7210 enabled_changes: enabled_changes.iter().cloned().collect(),
7211 unchanged,
7212 preview: false,
7213 restart_required: Vec::new(),
7216 capability_warnings: Vec::new(),
7217 module_actions: Vec::new(),
7218 earlier_pending: Vec::new(),
7219 expect: None,
7220 },
7221 changed,
7222 ))
7223 }
7224
7225 async fn handle_supervisor_set_enabled(
7226 &self,
7227 frame: Frame,
7228 module_id: String,
7229 enabled: bool,
7230 requester: Principal,
7231 ) -> Result<Vec<Frame>, RouterError> {
7232 let operation_lock = self.supervisor.operation_lock();
7233 let _operation_guard = operation_lock.lock().await;
7234 let Some(module) = self.supervisor.get(&module_id) else {
7235 return Ok(vec![control_error_frame(
7236 &frame,
7237 "unknown_module",
7238 format!("module_id '{module_id}' is not supervised"),
7239 )?]);
7240 };
7241
7242 self.route_outages.mark_operator_action(&module_id);
7245 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
7246 Ok(applied) => applied,
7247 Err(err) => {
7248 self.route_outages
7249 .operator_action_ended_unrefused(&module_id);
7250 return Ok(vec![control_error_frame(
7251 &frame,
7252 "target_unavailable",
7253 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
7254 )?]);
7255 }
7256 };
7257 if !applied {
7258 self.route_outages
7261 .operator_action_ended_unrefused(&module_id);
7262 }
7263
7264 self.capability_evaluator.wake_deadline_loop();
7265 self.refresh_capability_requirements();
7266 let response = ClientControlResponse::SupervisorAck { module_id, applied };
7267 Ok(vec![control_response_body_frame(
7268 &frame,
7269 &response,
7270 "ClientControlResponse::SupervisorAck",
7271 )?])
7272 }
7273
7274 async fn handle_supervisor_health_probe(
7275 &self,
7276 ctx: &RouteCtx,
7277 frame: Frame,
7278 module_id: String,
7279 ) -> Result<Vec<Frame>, RouterError> {
7280 self.refresh_capability_requirements();
7281 let Some(registration) = self
7282 .registry
7283 .get_module(&module_id)
7284 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
7285 else {
7286 return Ok(vec![control_error_frame(
7287 &frame,
7288 "unknown_module",
7289 format!("module_id '{module_id}' is not registered"),
7290 )?]);
7291 };
7292
7293 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
7306 {
7307 return Ok(vec![control_error_frame(
7308 &frame,
7309 "health_not_advertised",
7310 format!("module_id '{module_id}' did not advertise health.check"),
7311 )?]);
7312 }
7313
7314 self.observe_control_wait(
7315 ctx,
7316 Some(®istration.manifest.module_id),
7317 ops::SUPERVISOR_HEALTH_PROBE,
7318 async {
7319 let deadline = Instant::now() + self.health_probe_timeout;
7320 let pending = match self.forwarding.begin_module_control_rpc_for(
7321 &module_id,
7322 MODULE_CONTROL_OP_HEALTH_CHECK,
7323 deadline,
7324 ) {
7325 Ok(pending) => pending,
7326 Err(err) => {
7327 return Ok(vec![control_error_frame(
7328 &frame,
7329 forwarding_error_code(&err),
7330 err.to_string(),
7331 )?])
7332 }
7333 };
7334
7335 let PendingModuleControlRpc {
7336 endpoint,
7337 module_sink,
7338 negotiated_ver,
7339 corr: probe_corr,
7340 receiver,
7341 } = pending;
7342 let mut guard =
7343 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
7344 let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
7345 .map_err(|err| {
7346 RouterError::backend(
7347 0,
7348 frame.header.corr,
7349 format!("failed to encode health.check request: {err}"),
7350 )
7351 })?;
7352 let probe_frame = Frame::build_with_version(
7353 negotiated_ver,
7354 FrameType::Request,
7355 control_flags(),
7356 0,
7357 0,
7358 probe_corr,
7359 probe_body,
7360 )
7361 .map_err(RouterError::FrameBuild)?;
7362
7363 match timeout_at(deadline, module_sink.send(probe_frame)).await {
7366 Ok(Ok(())) => {}
7367 Ok(Err(err)) => {
7368 return Ok(vec![control_error_frame(
7369 &frame,
7370 "target_unavailable",
7371 err.to_string(),
7372 )?]);
7373 }
7374 Err(_) => {
7375 return Ok(vec![control_error_frame(
7376 &frame,
7377 "module_timeout",
7378 format!(
7379 "module_id '{module_id}' did not answer health.check within {:?}",
7380 self.health_probe_timeout
7381 ),
7382 )?]);
7383 }
7384 }
7385
7386 match timeout_at(deadline, receiver).await {
7387 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
7388 guard.disarm();
7389 let Some(report) = response.health_report() else {
7390 return Ok(vec![control_error_frame(
7391 &frame,
7392 "invalid_control_body",
7393 "health.check RPC returned a non-health response",
7394 )?]);
7395 };
7396 let HealthReport {
7401 status,
7402 detail,
7403 metrics,
7404 } = report;
7405 let capability_detail = self
7406 .capability_evaluator
7407 .required_problem_detail(&module_id);
7408 let response = ClientControlResponse::SupervisorHealthProbe {
7409 module_id,
7410 status,
7411 detail: append_capability_problem_detail(detail, capability_detail),
7412 metrics,
7413 };
7414 Ok(vec![control_response_body_frame(
7415 &frame,
7416 &response,
7417 "ClientControlResponse::SupervisorHealthProbe",
7418 )?])
7419 }
7420 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
7421 guard.disarm();
7422 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
7427 }
7428 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
7429 guard.disarm();
7430 Ok(vec![control_error_frame(
7431 &frame,
7432 "target_unavailable",
7433 message,
7434 )?])
7435 }
7436 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
7437 guard.disarm();
7438 Ok(vec![control_error_frame(
7439 &frame,
7440 "invalid_control_body",
7441 message,
7442 )?])
7443 }
7444 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
7445 guard.disarm();
7446 Ok(vec![control_error_frame(
7447 &frame,
7448 "invalid_control_body",
7449 format!("expected module-control op '{expected}', got '{actual}'"),
7450 )?])
7451 }
7452 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
7453 guard.disarm();
7454 Ok(vec![control_error_frame(
7455 &frame,
7456 "module_timeout",
7457 format!(
7458 "module_id '{module_id}' answered health.check after {:?}",
7459 self.health_probe_timeout
7460 ),
7461 )?])
7462 }
7463 Ok(Err(_)) => Ok(vec![control_error_frame(
7464 &frame,
7465 "target_unavailable",
7466 "health.check waiter was canceled before the module responded",
7467 )?]),
7468 Err(_) => Ok(vec![control_error_frame(
7469 &frame,
7470 "module_timeout",
7471 format!(
7472 "module_id '{module_id}' did not answer health.check within {:?}",
7473 self.health_probe_timeout
7474 ),
7475 )?]),
7476 }
7477 },
7478 )
7479 .await
7480 }
7481
7482 fn supervisor_status(
7483 &self,
7484 module_id: &str,
7485 corr: u64,
7486 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
7487 self.supervisor
7488 .get(module_id)
7489 .map(|module| {
7490 let warming = module.is_warming_for_control("status").map_err(|err| {
7491 RouterError::backend(
7492 0,
7493 corr,
7494 format!(
7495 "failed to read supervisor warming state for module_id '{module_id}': {err}"
7496 ),
7497 )
7498 })?;
7499 module.status_for_control("status").map_err(|err| {
7500 RouterError::backend(
7501 0,
7502 corr,
7503 format!(
7504 "failed to read supervisor status for module_id '{module_id}': {err}"
7505 ),
7506 )
7507 }).map(|status| (status, warming))
7508 })
7509 .transpose()
7510 }
7511
7512 fn guard_module_control_op(
7513 &self,
7514 frame: &Frame,
7515 module_id: &str,
7516 op: &str,
7517 ) -> Result<Option<Frame>, RouterError> {
7518 if self.module_grants_op(module_id, op, frame.header.corr)? {
7519 return Ok(None);
7520 }
7521
7522 Ok(Some(control_error_frame(
7523 frame,
7524 "op_not_allowed",
7525 format!("module_id '{module_id}' did not grant control op '{op}'"),
7526 )?))
7527 }
7528
7529 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
7530 let Some(registration) = self
7531 .registry
7532 .get_module(module_id)
7533 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
7534 else {
7535 return Ok(false);
7536 };
7537 Ok(module_registration_grants_op(®istration.control_ops, op))
7538 }
7539
7540 fn handle_status_update(
7541 &self,
7542 endpoint: ModuleEndpointId,
7543 frame: Frame,
7544 ) -> Result<Vec<Frame>, RouterError> {
7545 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
7546 Ok(update) => update,
7547 Err(err) => {
7548 if is_known_module_push_op(&frame.body) {
7553 return Ok(vec![control_error_frame(
7554 &frame,
7555 "invalid_control_body",
7556 format!("malformed module control push body: {err}"),
7557 )?]);
7558 }
7559 return Ok(Vec::new());
7560 }
7561 };
7562
7563 match update {
7564 ModuleControlPush::RouteStatus {
7565 route_channel,
7566 route_epoch,
7567 status,
7568 } => {
7569 self.forwarding
7570 .cache_status(endpoint, route_channel, route_epoch, status)
7571 .map_err(RouterError::Forwarding)?;
7572 }
7573 }
7574 Ok(Vec::new())
7575 }
7576
7577 fn handle_route_poll(
7578 &self,
7579 ctx: &RouteCtx,
7580 frame: Frame,
7581 route_channel: u16,
7582 route_epoch: u32,
7583 kind: PollKind,
7584 ) -> Result<Vec<Frame>, RouterError> {
7585 let snapshot = self
7586 .forwarding
7587 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7588 .map_err(RouterError::Forwarding)?;
7589 let response = match (kind, snapshot) {
7590 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7591 ClientControlResponse::RoutePoll {
7592 route_channel,
7593 route_epoch,
7594 status,
7595 live: None,
7596 }
7597 }
7598 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7599 route_channel,
7600 route_epoch,
7601 status: None,
7602 live: None,
7603 },
7604 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7605 let live = self
7631 .process_liveness
7632 .as_ref()
7633 .and_then(|source| source.process_live(&module_id))
7634 .unwrap_or(true);
7635 ClientControlResponse::RoutePoll {
7636 route_channel,
7637 route_epoch,
7638 status: None,
7639 live: Some(live),
7640 }
7641 }
7642 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7643 route_channel,
7644 route_epoch,
7645 status: None,
7646 live: Some(false),
7647 },
7648 };
7649
7650 Ok(vec![control_response_body_frame(
7651 &frame,
7652 &response,
7653 "ClientControlResponse::RoutePoll",
7654 )?])
7655 }
7656
7657 pub(crate) fn observe_module_control_completion(
7658 &self,
7659 completion: ModuleControlRpcCompletion,
7660 ) -> bool {
7661 match completion {
7662 ModuleControlRpcCompletion::Unknown => false,
7663 ModuleControlRpcCompletion::Settled => true,
7664 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7665 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7666 info!(
7667 module_id = %module_id,
7668 latency_ms,
7669 "late health.check answer proves the module is alive"
7670 );
7671 match self
7672 .supervisor
7673 .record_late_health_answer(&module_id, latency_ms)
7674 {
7675 Ok(true) => {}
7676 Ok(false) => debug!(
7677 module_id = %module_id,
7678 latency_ms,
7679 "late health.check answer has no active supervisor snapshot"
7680 ),
7681 Err(err) => warn!(
7682 module_id = %module_id,
7683 latency_ms,
7684 error = %err,
7685 "failed to record late health.check answer"
7686 ),
7687 }
7688 true
7689 }
7690 }
7691 }
7692
7693 fn refuse_to_end_module_connection_for_a_client(
7714 &self,
7715 module_connection_id: ConnectionId,
7716 corr: u64,
7717 err: ForwardingError,
7718 ) -> Result<(), RouterError> {
7719 if let ForwardingError::ConnectionClosing { connection_id } = err {
7720 if connection_id != module_connection_id {
7721 warn!(
7722 module_connection_id = module_connection_id.get(),
7723 client_connection_id = connection_id.get(),
7724 corr,
7725 "dropping a route.bind response for a closing client; the module connection keeps serving"
7726 );
7727 return Ok(());
7728 }
7729 }
7730 Err(RouterError::Forwarding(err))
7731 }
7732
7733 fn handle_module_relay_response(
7734 &self,
7735 connection_id: ConnectionId,
7736 frame: Frame,
7737 ) -> Result<Vec<Frame>, RouterError> {
7738 let mut secondary_error = None;
7739 let outcome = match frame.header.ty {
7740 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7741 Ok(probe) if probe.op == "route.bind" => {
7742 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7743 Ok(ModuleControlResponse::RouteBindAck {}) => {
7744 RouteBindRelayOutcome::Accepted
7745 }
7746 Ok(other) => {
7747 let message =
7748 format!("route.bind response carried unexpected body: {other:?}");
7749 secondary_error = Some(control_error_frame(
7750 &frame,
7751 "invalid_control_body",
7752 message.clone(),
7753 )?);
7754 RouteBindRelayOutcome::ModuleGone(message)
7755 }
7756 Err(err) => {
7757 let message = format!("malformed route.bind response body: {err}");
7758 secondary_error = Some(control_error_frame(
7759 &frame,
7760 "invalid_control_body",
7761 message.clone(),
7762 )?);
7763 RouteBindRelayOutcome::ModuleGone(message)
7764 }
7765 }
7766 }
7767 Ok(probe) => {
7768 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7769 {
7770 Ok(response) => ModuleControlRpcOutcome::Response(response),
7771 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7772 "malformed {} response body: {err}",
7773 probe.op
7774 )),
7775 };
7776 let completion = self
7777 .forwarding
7778 .complete_module_control_rpc(
7779 connection_id,
7780 frame.header.corr,
7781 Some(&probe.op),
7782 outcome,
7783 )
7784 .map_err(RouterError::Forwarding)?;
7785 if !self.observe_module_control_completion(completion) {
7786 debug!(
7787 connection_id = connection_id.get(),
7788 corr = frame.header.corr,
7789 op = %probe.op,
7790 "dropping late or unknown module-control RPC response"
7791 );
7792 }
7793 return Ok(Vec::new());
7794 }
7795 Err(err) => {
7796 if let Some(expected_op) = self
7797 .forwarding
7798 .pending_module_control_op(connection_id, frame.header.corr)
7799 .map_err(RouterError::Forwarding)?
7800 {
7801 let completion = self
7802 .forwarding
7803 .complete_module_control_rpc(
7804 connection_id,
7805 frame.header.corr,
7806 None,
7807 ModuleControlRpcOutcome::MalformedResponse(format!(
7808 "malformed {expected_op} response body: {err}"
7809 )),
7810 )
7811 .map_err(RouterError::Forwarding)?;
7812 if !self.observe_module_control_completion(completion) {
7813 debug!(
7814 connection_id = connection_id.get(),
7815 corr = frame.header.corr,
7816 "dropping late malformed module-control RPC response"
7817 );
7818 }
7819 return Ok(Vec::new());
7820 }
7821 let message = format!("malformed route.bind response body: {err}");
7822 secondary_error = Some(control_error_frame(
7823 &frame,
7824 "invalid_control_body",
7825 message.clone(),
7826 )?);
7827 RouteBindRelayOutcome::ModuleGone(message)
7828 }
7829 },
7830 FrameType::Error => {
7831 if self
7832 .forwarding
7833 .pending_module_control_op(connection_id, frame.header.corr)
7834 .map_err(RouterError::Forwarding)?
7835 .is_some()
7836 {
7837 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7838 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7839 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7840 "malformed module-control ERROR body: {err}"
7841 )),
7842 };
7843 let completion = self
7844 .forwarding
7845 .complete_module_control_rpc(
7846 connection_id,
7847 frame.header.corr,
7848 None,
7849 outcome,
7850 )
7851 .map_err(RouterError::Forwarding)?;
7852 if !self.observe_module_control_completion(completion) {
7853 debug!(
7854 connection_id = connection_id.get(),
7855 corr = frame.header.corr,
7856 "dropping late or unknown module-control RPC error"
7857 );
7858 }
7859 return Ok(Vec::new());
7860 }
7861 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7862 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7863 Err(err) => {
7864 let message = format!("malformed route.bind ERROR body: {err}");
7865 secondary_error = Some(control_error_frame(
7866 &frame,
7867 "invalid_control_body",
7868 message.clone(),
7869 )?);
7870 RouteBindRelayOutcome::ModuleGone(message)
7871 }
7872 }
7873 }
7874 ty => {
7875 return Ok(vec![control_error_frame(
7876 &frame,
7877 "unsupported_control_frame",
7878 format!("unsupported module channel-0 frame {ty:?}"),
7879 )?])
7880 }
7881 };
7882
7883 let settled =
7884 self.forwarding
7885 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7886 let completion = match settled {
7887 Ok(completion) => completion,
7888 Err(err) => {
7889 self.refuse_to_end_module_connection_for_a_client(
7890 connection_id,
7891 frame.header.corr,
7892 err,
7893 )?;
7894 return Ok(secondary_error.into_iter().collect());
7895 }
7896 };
7897 if let Some(target) = completion.abandoned.as_ref() {
7898 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7899 }
7900 if !completion.settled {
7901 debug!(
7902 connection_id = connection_id.get(),
7903 corr = frame.header.corr,
7904 frame_type = ?frame.header.ty,
7905 "dropping late or unknown route.bind relay response"
7906 );
7907 }
7908 Ok(secondary_error.into_iter().collect())
7909 }
7910
7911 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7912 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7913 self.cleanup_connection_with_end_reason(
7917 connection_id,
7918 RegistrationEndReason::ExplicitGoodbye,
7919 )
7920 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7921 Ok(Vec::new())
7922 }
7923}
7924
7925impl Default for ControlHandler {
7926 fn default() -> Self {
7927 Self::new(Arc::new(Registry::default()))
7928 }
7929}
7930
7931impl crate::supervise::SwapPromotionObserver for ControlHandler {
7932 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7933 self.apply_registration_capabilities(registration);
7934 }
7935}
7936
7937fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7938 CapabilityRequirementStatus {
7939 consumer: status.consumer,
7940 capability: status.capability,
7941 need: match status.need {
7942 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7943 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7944 },
7945 verdict: status.verdict.as_str().to_string(),
7946 episode_seq: status.episode_seq,
7947 config_satisfiable: status.config_satisfiable,
7948 runtime_available: status.runtime_available,
7949 detail: status.detail,
7950 }
7951}
7952
7953fn append_capability_problem_detail(
7954 detail: Option<String>,
7955 capability_detail: Option<String>,
7956) -> Option<String> {
7957 match (detail, capability_detail) {
7958 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7959 (Some(detail), None) => Some(detail),
7960 (None, Some(capability_detail)) => Some(capability_detail),
7961 (None, None) => None,
7962 }
7963}
7964
7965fn subc_ops() -> Vec<String> {
7966 SUBC_CONTROL_OPS
7967 .iter()
7968 .map(|op| (*op).to_string())
7969 .collect()
7970}
7971
7972fn module_subc_ops() -> Vec<String> {
7973 SUBC_CONTROL_OPS
7974 .iter()
7975 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7976 .map(|op| (*op).to_string())
7977 .collect()
7978}
7979
7980#[cfg(test)]
7981fn module_baseline_control_ops() -> Vec<String> {
7982 MODULE_BASELINE_CONTROL_OPS
7983 .iter()
7984 .map(|op| (*op).to_string())
7985 .collect()
7986}
7987
7988fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7989 let mut seen = HashSet::new();
7990 let mut effective = Vec::new();
7991 for op in MODULE_BASELINE_CONTROL_OPS {
7992 if seen.insert((*op).to_string()) {
7993 effective.push((*op).to_string());
7994 }
7995 }
7996 for op in declared.unwrap_or_default() {
7997 if seen.insert(op.clone()) {
7998 effective.push(op);
7999 }
8000 }
8001 effective
8002}
8003
8004fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
8005 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
8006}
8007
8008fn target_module_id(target: &RouteTarget) -> &str {
8009 match target {
8010 RouteTarget::ToolProvider { module_id }
8011 | RouteTarget::ManagementSurface { module_id }
8012 | RouteTarget::InternalService { module_id, .. } => module_id,
8013 }
8014}
8015
8016fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
8017 roles.iter().any(|role| match (target, role) {
8018 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
8019 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
8020 (
8021 RouteTarget::InternalService { service_id, .. },
8022 ProviderRole::InternalService {
8023 service_id: provided,
8024 ..
8025 },
8026 ) => service_id == provided,
8027 _ => false,
8028 })
8029}
8030
8031fn is_routable_role(role: &ProviderRole) -> bool {
8032 matches!(
8033 role,
8034 ProviderRole::ToolProvider { .. }
8035 | ProviderRole::ManagementSurface { .. }
8036 | ProviderRole::InternalService { .. }
8037 )
8038}
8039
8040#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8041enum ControlRequestBodyError {
8042 UnknownOp,
8043 InvalidBody,
8044}
8045
8046#[derive(Debug, Deserialize)]
8047struct ControlOpProbe {
8048 op: String,
8049}
8050
8051const MODULE_PUSH_OPS: &[&str] = &["route.status"];
8054
8055fn is_known_module_push_op(body: &[u8]) -> bool {
8056 serde_json::from_slice::<ControlOpProbe>(body)
8057 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
8058 .unwrap_or(false)
8059}
8060
8061fn is_known_module_request_op(body: &[u8]) -> bool {
8062 serde_json::from_slice::<ControlOpProbe>(body)
8063 .map(|probe| is_module_to_subc_op(&probe.op))
8064 .unwrap_or(false)
8065}
8066
8067fn is_module_to_subc_op(op: &str) -> bool {
8068 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
8069}
8070
8071fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
8072 debug!(
8073 op = %op,
8074 connection_id = connection_id.get(),
8075 corr,
8076 "control dispatch"
8077 );
8078}
8079
8080fn log_slow_control_dispatch(
8081 dispatch_started_at: Option<StdInstant>,
8082 op: &'static str,
8083 connection_id: ConnectionId,
8084 corr: u64,
8085 route_open_timings: Option<&RouteOpenTimings>,
8086) {
8087 let Some(dispatch_started_at) = dispatch_started_at else {
8088 return;
8089 };
8090 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
8092 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
8093 if let Some(timings) = route_open_timings {
8094 warn!(
8095 op = %op,
8096 connection_id = connection_id.get(),
8097 corr,
8098 elapsed_ms = elapsed.as_millis() as u64,
8099 module_id = ?timings.module_id,
8100 principal = %timings.principal,
8101 admission_ms = timings.admission.as_millis() as u64,
8102 project_root_ms = timings.project_root.as_millis() as u64,
8103 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
8104 bind_send_ms = timings.bind_send.as_millis() as u64,
8105 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
8106 "slow control dispatch"
8107 );
8108 return;
8109 }
8110 warn!(
8111 op = %op,
8112 connection_id = connection_id.get(),
8113 corr,
8114 elapsed_ms = elapsed.as_millis() as u64,
8115 "slow control dispatch"
8116 );
8117 }
8118}
8119
8120#[cfg(test)]
8122type TestCallerLookup =
8123 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
8124
8125fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
8127 match request {
8128 ClientControlRequest::SupervisorRestart { module_id, .. } => {
8129 Some(PrivilegedOperation::Restart { module_id })
8130 }
8131 ClientControlRequest::SupervisorSwap { module_id, .. } => {
8132 Some(PrivilegedOperation::Swap { module_id })
8133 }
8134 ClientControlRequest::SupervisorReload { module_id } => {
8135 Some(PrivilegedOperation::Reload { module_id })
8136 }
8137 ClientControlRequest::SupervisorRescan { preview } => {
8138 if *preview {
8139 None
8140 } else {
8141 Some(PrivilegedOperation::Rescan)
8142 }
8143 }
8144 ClientControlRequest::SupervisorRescanChecked {
8149 preview,
8150 candidate,
8151 expect,
8152 } => (!*preview && candidate.is_none() && expect.is_some())
8153 .then_some(PrivilegedOperation::Rescan),
8154 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
8155 Some(PrivilegedOperation::SetEnabled {
8156 module_id,
8157 enabled: *enabled,
8158 })
8159 }
8160 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
8161 Some(PrivilegedOperation::ReleaseReserved { module_id })
8162 }
8163 ClientControlRequest::ServerDescribe {}
8164 | ClientControlRequest::DaemonShutdownIntent { .. }
8167 | ClientControlRequest::CatalogList { .. }
8168 | ClientControlRequest::RouteOpen { .. }
8169 | ClientControlRequest::RoutePoll { .. }
8170 | ClientControlRequest::SupervisorList {}
8171 | ClientControlRequest::SupervisorHealth {}
8172 | ClientControlRequest::SupervisorHealthProbe { .. }
8173 | ClientControlRequest::SupervisorRoutes { .. }
8174 | ClientControlRequest::SupervisorProvenance { .. }
8175 | ClientControlRequest::SupervisorStderrTail { .. }
8176 | ClientControlRequest::SupervisorTerminals { .. }
8177 | ClientControlRequest::SupervisorSpawnSnapshot {}
8178 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
8179 }
8180}
8181
8182fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
8183 match request {
8184 ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
8185 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
8186 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
8187 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
8188 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
8189 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
8190 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
8191 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
8192 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
8193 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
8194 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
8195 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
8196 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
8197 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
8198 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
8199 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
8200 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
8201 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
8202 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
8203 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
8204 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
8205 }
8206}
8207
8208struct RouteOpenCaller {
8216 principal: String,
8220 scope: String,
8223 harness: String,
8226}
8227
8228impl RouteOpenCaller {
8229 fn new(
8230 principal: Option<&Principal>,
8231 scope: Option<&ScopeSelector>,
8232 identity: &BindIdentity,
8233 ) -> Self {
8234 Self {
8235 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
8236 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
8237 harness: bounded_log_text(&identity.harness),
8238 }
8239 }
8240
8241 fn unknown() -> Self {
8244 Self {
8245 principal: "unknown".to_string(),
8246 scope: "unknown".to_string(),
8247 harness: "unknown".to_string(),
8248 }
8249 }
8250}
8251
8252const MAX_LOGGED_TEXT_CHARS: usize = 64;
8255
8256fn bounded_log_text(value: &str) -> String {
8260 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
8261 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
8262 bounded.push('…');
8263 }
8264 format!("{bounded:?}")
8265}
8266
8267fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
8273 use subc_protocol::scope::ScopeKind;
8274 let kind = match stamp.kind {
8275 ScopeKind::Head => "head",
8276 ScopeKind::Worker => "worker",
8277 ScopeKind::Ephemeral => "ephemeral",
8278 ScopeKind::Federated => "federated",
8279 _ => "unknown",
8280 };
8281 format!(
8282 "{kind}/{}/{}@{}",
8283 principal_label(&stamp.owner),
8284 bounded_log_text(&stamp.scope_ref),
8285 stamp.scope_epoch
8286 )
8287}
8288
8289fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
8296 let epoch = selector
8297 .scope_epoch
8298 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
8299 format!(
8300 "requested/{}/{}@{epoch}",
8301 bounded_log_text(&principal_label(&selector.owner)),
8302 bounded_log_text(&selector.scope_ref),
8303 )
8304}
8305
8306fn principal_label(principal: &Principal) -> String {
8307 match principal {
8308 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
8309 Principal::Direct => "direct".to_string(),
8310 other => format!("{other:?}"),
8311 }
8312}
8313
8314fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
8315 match request {
8316 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
8317 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
8318 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
8319 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
8320 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
8321 ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
8322 _ => "unsupported",
8323 }
8324}
8325
8326fn parse_client_control_request(
8327 body: &[u8],
8328) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
8329 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
8330 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8331 Ok(probe)
8332 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
8333 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
8334 {
8335 ControlRequestBodyError::InvalidBody
8336 }
8337 Ok(_) => ControlRequestBodyError::UnknownOp,
8338 Err(_) => ControlRequestBodyError::InvalidBody,
8339 };
8340 (err, classification)
8341 })
8342}
8343
8344fn parse_module_control_request_from_module(
8345 body: &[u8],
8346) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
8347 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
8348 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8349 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
8350 Ok(_) => ControlRequestBodyError::UnknownOp,
8351 Err(_) => ControlRequestBodyError::InvalidBody,
8352 };
8353 (err, classification)
8354 })
8355}
8356
8357#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
8358enum ProviderRoleKind {
8359 ToolProvider,
8360 PipelineStage,
8361 ManagementSurface,
8362 InternalService,
8363}
8364
8365fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
8366 match role {
8367 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
8368 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
8369 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
8370 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
8371 }
8372}
8373
8374fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
8375 roles.iter().map(provider_role_kind).collect()
8376}
8377
8378const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
8381
8382#[derive(Debug, Default, PartialEq, Eq)]
8384struct ScopeOutcomeCounts {
8385 created: usize,
8386 replaced: usize,
8387 updated: usize,
8388 unchanged: usize,
8389 refused: usize,
8390}
8391
8392impl ScopeOutcomeCounts {
8393 fn of(results: &[ScopeRecordResult]) -> Self {
8394 let mut counts = Self::default();
8395 for result in results {
8396 let slot = match result.outcome {
8397 ScopeRecordOutcome::Created => &mut counts.created,
8398 ScopeRecordOutcome::Replaced => &mut counts.replaced,
8399 ScopeRecordOutcome::Updated => &mut counts.updated,
8400 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
8401 ScopeRecordOutcome::Refused => &mut counts.refused,
8402 };
8403 *slot += 1;
8404 }
8405 counts
8406 }
8407}
8408
8409#[cfg(test)]
8410mod scope_outcome_count_tests {
8411 use super::*;
8412
8413 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
8414 ScopeRecordResult {
8415 scope_ref: "r".to_string(),
8416 scope_epoch: 1,
8417 outcome,
8418 code: None,
8419 message: None,
8420 version: None,
8421 parent_state: None,
8422 }
8423 }
8424
8425 #[test]
8428 fn every_outcome_is_counted_in_its_own_field() {
8429 let results = [
8430 result(ScopeRecordOutcome::Created),
8431 result(ScopeRecordOutcome::Created),
8432 result(ScopeRecordOutcome::Replaced),
8433 result(ScopeRecordOutcome::Updated),
8434 result(ScopeRecordOutcome::Unchanged),
8435 result(ScopeRecordOutcome::Refused),
8436 result(ScopeRecordOutcome::Refused),
8437 result(ScopeRecordOutcome::Refused),
8438 ];
8439 assert_eq!(
8440 ScopeOutcomeCounts::of(&results),
8441 ScopeOutcomeCounts {
8442 created: 2,
8443 replaced: 1,
8444 updated: 1,
8445 unchanged: 1,
8446 refused: 3,
8447 }
8448 );
8449 }
8450}
8451
8452fn capability_census_trigger(
8456 old: Option<&CapabilityDeclarations>,
8457 new: Option<&CapabilityDeclarations>,
8458) -> bool {
8459 let old_provides = old
8460 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
8461 .unwrap_or_default();
8462 let old_denies = old
8463 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
8464 .unwrap_or_default();
8465 let new = new.cloned().unwrap_or(CapabilityDeclarations {
8466 provides: Vec::new(),
8467 requires: Vec::new(),
8468 must_never_reach: Vec::new(),
8469 });
8470
8471 new.provides
8472 .iter()
8473 .any(|capability| !old_provides.contains(capability))
8474 || new
8475 .must_never_reach
8476 .iter()
8477 .any(|capability| !old_denies.contains(capability))
8478}
8479
8480fn denied_capability<'a>(
8483 opening_manifest: &'a ModuleManifest,
8484 target_manifest: &ModuleManifest,
8485) -> Option<&'a str> {
8486 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
8487 let target_capabilities = target_manifest.capabilities.as_ref()?;
8488 opening_capabilities
8489 .must_never_reach
8490 .iter()
8491 .find(|denied| {
8492 target_capabilities
8493 .provides
8494 .iter()
8495 .any(|provided| provided == *denied)
8496 })
8497 .map(String::as_str)
8498}
8499
8500fn catalog_update_frozen_field_message(
8501 registered: &ModuleManifest,
8502 provides: &[ProviderRole],
8503) -> Option<String> {
8504 let old_has_provides = !registered.provides.is_empty();
8505 let new_has_provides = !provides.is_empty();
8506 if old_has_provides != new_has_provides {
8507 return Some(format!(
8508 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
8509 registered.module_id
8510 ));
8511 }
8512
8513 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
8514 return Some(format!(
8515 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
8516 registered.module_id
8517 ));
8518 }
8519
8520 let registered_concurrency = manifest_concurrency(registered);
8521 let mut candidate = registered.clone();
8522 candidate.provides = provides.to_vec();
8523 let candidate_concurrency = manifest_concurrency(&candidate);
8524 if candidate_concurrency != registered_concurrency {
8525 return Some(format!(
8526 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
8527 registered.module_id, registered_concurrency, candidate_concurrency
8528 ));
8529 }
8530
8531 None
8534}
8535
8536fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
8537 manifest.provides.iter().any(is_routable_role)
8538}
8539
8540fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
8546 manifest
8547 .provides
8548 .iter()
8549 .find_map(|provider| match provider {
8550 ProviderRole::ToolProvider { concurrency, .. }
8551 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
8552 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
8553 })
8554 .unwrap_or(Concurrency::ModuleManaged)
8555}
8556
8557fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
8564 let has_management_surface = manifest
8565 .provides
8566 .iter()
8567 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
8568 if !has_management_surface {
8569 return false;
8570 }
8571 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
8572 return false;
8573 };
8574 let Some(provides) = raw
8575 .get("manifest")
8576 .and_then(|manifest| manifest.get("provides"))
8577 .and_then(serde_json::Value::as_array)
8578 else {
8579 return false;
8580 };
8581 provides.iter().any(|role| {
8586 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8587 && role.get("concurrency").is_none()
8588 })
8589}
8590
8591fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8592 if peer_version != PROTOCOL_VERSION {
8593 return Err(format!(
8594 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8595 ));
8596 }
8597 Ok(PROTOCOL_VERSION)
8598}
8599
8600fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8601 Frame::build_with_version(
8602 response_version(frame),
8603 FrameType::Pong,
8604 frame.header.flags,
8605 0,
8606 0,
8607 frame.header.corr,
8608 Vec::new(),
8609 )
8610 .map_err(RouterError::FrameBuild)
8611}
8612
8613pub(crate) fn control_error_frame(
8614 frame: &Frame,
8615 code: &'static str,
8616 message: impl Into<String>,
8617) -> Result<Frame, RouterError> {
8618 control_error_body_frame(
8619 frame,
8620 ErrorBody {
8621 code: code.to_string(),
8622 message: message.into(),
8623 detail: None,
8624 },
8625 )
8626}
8627
8628fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8636 control_error_frame_with_flags(frame, error, daemon_error_flags())
8637}
8638
8639fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8645 control_error_frame_with_flags(frame, error, control_flags())
8646}
8647
8648fn control_error_frame_with_flags(
8649 frame: &Frame,
8650 error: ErrorBody,
8651 flags: Flags,
8652) -> Result<Frame, RouterError> {
8653 let body = serde_json::to_vec(&error).map_err(|err| {
8654 RouterError::backend(
8655 0,
8656 frame.header.corr,
8657 format!("failed to encode control ERROR: {err}"),
8658 )
8659 })?;
8660
8661 Frame::build_with_version(
8662 response_version(frame),
8663 FrameType::Error,
8664 flags,
8665 0,
8666 0,
8667 frame.header.corr,
8668 body,
8669 )
8670 .map_err(RouterError::FrameBuild)
8671}
8672
8673pub(crate) fn control_response_body_frame<T: Serialize>(
8674 frame: &Frame,
8675 reply: &T,
8676 label: &'static str,
8677) -> Result<Frame, RouterError> {
8678 let body = serde_json::to_vec(reply).map_err(|err| {
8679 RouterError::backend(
8680 0,
8681 frame.header.corr,
8682 format!("failed to encode {label}: {err}"),
8683 )
8684 })?;
8685
8686 Frame::build_with_version(
8687 response_version(frame),
8688 FrameType::Response,
8689 control_flags(),
8690 0,
8691 0,
8692 frame.header.corr,
8693 body,
8694 )
8695 .map_err(RouterError::FrameBuild)
8696}
8697
8698fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8728 match err {
8729 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8730 ForwardingError::NoModuleConnection => "target_unavailable",
8731 ForwardingError::ModuleReloading { .. } => "module_reloading",
8732 ForwardingError::ClientRouteChannelExhausted { .. }
8733 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8734 ForwardingError::StaleModuleEndpoint
8735 | ForwardingError::UnknownReservation { .. }
8736 | ForwardingError::ConnectionClosing { .. }
8737 | ForwardingError::ClientEgressClosed { .. }
8738 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8739 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8742 ForwardingError::RelayCorrelationExhausted
8743 | ForwardingError::RouteOpenBuild(_)
8744 | ForwardingError::Poisoned => "forwarding_error",
8745 }
8746}
8747
8748fn response_version(frame: &Frame) -> u8 {
8749 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8750 frame.header.ver
8751 } else {
8752 PROTOCOL_VERSION
8753 }
8754}
8755
8756fn control_flags() -> Flags {
8757 Flags::new(false, Priority::Passive, false)
8758}
8759
8760fn send_goodbye_target_best_effort(
8765 counters: &DaemonCounters,
8766 target: &GoodbyeTarget,
8767 context: &'static str,
8768) {
8769 let Ok(frame) = Frame::build_with_version(
8770 target.negotiated_ver,
8771 FrameType::Goodbye,
8772 control_flags(),
8773 target.channel,
8774 target.epoch,
8775 0,
8776 Vec::new(),
8777 ) else {
8778 return;
8779 };
8780 crate::forwarding::send_module_route_goodbye(
8781 counters,
8782 &target.sink,
8783 frame,
8784 target.module_id.as_deref(),
8785 context,
8786 );
8787}
8788
8789pub(crate) fn send_route_control_pushes(
8790 forwarding: &ForwardingTable,
8791 routes: Vec<EndpointRoute>,
8792 push: ClientControlPush,
8793) {
8794 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8795 for route in routes {
8796 let target = route.goodbye_target;
8797 if let Some((existing, channels)) = targets
8798 .iter_mut()
8799 .find(|(existing, _)| existing.connection_id == target.connection_id)
8800 {
8801 debug_assert_eq!(
8802 existing.negotiated_ver, target.negotiated_ver,
8803 "one connection cannot negotiate multiple frame versions"
8804 );
8805 if !channels.contains(&target.channel) {
8806 channels.push(target.channel);
8807 }
8808 continue;
8809 }
8810 let channel = target.channel;
8811 targets.push((target, vec![channel]));
8812 }
8813 for (target, mut channels) in targets {
8814 channels.sort_unstable();
8815 let mut push = push.clone();
8816 match &mut push {
8817 ClientControlPush::RouteClosing {
8818 channels: covered, ..
8819 }
8820 | ClientControlPush::RouteClosed {
8821 channels: covered, ..
8822 } => *covered = channels,
8823 }
8824 let body = match serde_json::to_vec(&push) {
8825 Ok(body) => body,
8826 Err(err) => {
8827 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8828 continue;
8829 }
8830 };
8831 let frame = match Frame::build_with_version(
8832 target.negotiated_ver,
8833 FrameType::Push,
8834 control_flags(),
8835 0,
8836 0,
8837 0,
8838 body.clone(),
8839 ) {
8840 Ok(frame) => frame,
8841 Err(err) => {
8842 warn!(
8843 route_channel = target.channel,
8844 error = %err,
8845 "failed to build route lifecycle control PUSH frame"
8846 );
8847 continue;
8848 }
8849 };
8850 if let Err(err) = target.sink.try_send(frame) {
8851 if target.close_on_delivery_failure() {
8852 warn!(
8853 target_connection_id = target.connection_id.get(),
8854 route_channel = target.channel,
8855 error = %err,
8856 "route lifecycle control PUSH was not delivered to client; closing target connection"
8857 );
8858 let _ = forwarding.escalate_client_delivery_failure(
8859 target.connection_id,
8860 target.channel,
8861 target.epoch,
8862 CloseReason::new(
8863 "route_lifecycle_push_delivery_failed",
8864 format!(
8865 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8866 target.channel
8867 ),
8868 ),
8869 crate::forwarding::UndeliveredFrame {
8870 module_id: target.module_id.as_deref(),
8871 sink: &target.sink,
8872 },
8873 );
8874 }
8875 }
8876 }
8877}
8878
8879#[cfg(test)]
8880mod tests {
8881 include!("control/agent_commands_tests.rs");
8882 include!("control/resource_revision_tests.rs");
8883 #[tokio::test]
8884 async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
8885 let registry = Arc::new(Registry::default());
8886 let handler = ControlHandler::new(registry);
8887 assert_eq!(
8888 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8889 None
8890 );
8891 let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
8892 hello_via_sink(
8893 &handler,
8894 &ctx,
8895 &mut rx,
8896 hello_frame("daemon", PROTOCOL_VERSION, 1),
8897 )
8898 .await;
8899 assert_eq!(
8900 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8901 Some("daemon".into())
8902 );
8903 assert_eq!(
8904 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
8905 None
8906 );
8907 assert_eq!(
8908 handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
8909 None,
8910 "a registered module is not necessarily supervised"
8911 );
8912 assert_eq!(
8913 handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
8914 None,
8915 "aggregate provenance has no single target"
8916 );
8917 assert_eq!(
8918 handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
8919 None
8920 );
8921 }
8922
8923 mod peer_manifest {
8924 include!("peer_manifest_tests.rs");
8925 }
8926 mod activity_read {
8927 include!("activity_read_tests.rs");
8928 }
8929
8930 mod lifecycle_activity {
8931 include!("control_lifecycle_tests.rs");
8932 }
8933 #[cfg(unix)]
8934 #[tokio::test]
8935 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8936 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8937 let path = dir.join("subc.jsonc");
8938 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8939 "program":"/bin/sleep","args":["60"],"protocol":"none",
8940 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8941 }}}).to_string()).unwrap();
8942 let mut configured = crate::daemon_config::load(&path)
8943 .unwrap()
8944 .unwrap()
8945 .modules
8946 .pop()
8947 .unwrap();
8948 let registry = std::sync::Arc::new(crate::Registry::default());
8949 let handle = crate::SupervisorHandle::new();
8950 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8951 .with_handle(handle.clone());
8952 let module = supervisor
8953 .supervise_configured_with_health(
8954 configured.module_spec(),
8955 true,
8956 configured.health.clone(),
8957 None,
8958 configured.restart,
8959 )
8960 .unwrap();
8961 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8962 let before = module.status().unwrap().pid;
8963 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8964 configured.health.cadence = std::time::Duration::from_secs(3600);
8965 let health_only = handler
8966 .reconcile_supervised_modules(
8967 &supervisor,
8968 vec![configured.clone()],
8969 false,
8970 Principal::Direct,
8971 )
8972 .await
8973 .unwrap();
8974 assert!(
8975 health_only.0.changed_pending_reload.is_empty(),
8976 "health policy is already applied live"
8977 );
8978 assert_eq!(module.status().unwrap().pid, before);
8979 assert_eq!(
8980 module.configuration().unwrap().1.http,
8981 configured.health.http
8982 );
8983 configured.args = vec!["61".into()];
8984 let launch = handler
8985 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8986 .await
8987 .unwrap();
8988 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8989 assert_eq!(
8990 module.status().unwrap().pid,
8991 before,
8992 "a launch edit is stored until reload"
8993 );
8994 module.drain().await.unwrap();
8995 }
8996 use cortexkit_test_support::ScratchDir;
8997 use std::{
8998 collections::BTreeMap,
8999 fmt,
9000 path::PathBuf,
9001 sync::{Arc, Mutex},
9002 time::Duration,
9003 };
9004
9005 use serde_json::{json, Value};
9006 use subc_protocol::{
9007 manifest::{
9008 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
9009 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
9010 },
9011 session::HealthStatus,
9012 FrameType,
9013 };
9014
9015 use super::*;
9016 use crate::{
9017 forwarding::{DataRoute, DataRouteState},
9018 registry::ChannelState,
9019 router::FrameSink,
9020 stderr_tail::DEFAULT_MAX_LINE_BYTES,
9021 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
9022 RouteCtx, Router,
9023 };
9024 use tokio::{
9025 sync::mpsc,
9026 time::{sleep, Instant},
9027 };
9028 use tracing::{
9029 field::{Field, Visit},
9030 Event, Subscriber,
9031 };
9032 use tracing_subscriber::{layer::Context, prelude::*, Layer};
9033
9034 fn fake_aft_stub_path() -> PathBuf {
9053 let mut path = std::env::current_exe().expect("current_exe available in tests");
9054 path.pop(); path.pop(); path.push(if cfg!(windows) {
9057 "fake-aft-stub.exe"
9058 } else {
9059 "fake-aft-stub"
9060 });
9061 assert!(
9062 path.exists(),
9063 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
9064 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
9065 path.display()
9066 );
9067 path
9068 }
9069
9070 fn client_retries(code: &str) -> bool {
9075 subc_protocol::error_codes::is_retryable_route_open(code)
9076 }
9077
9078 #[test]
9089 fn retryability_of_forwarding_codes_matches_the_failure() {
9090 let transient = [
9093 ForwardingError::NoModuleConnection,
9094 ForwardingError::ModuleReloading {
9095 module_id: "m".into(),
9096 },
9097 ForwardingError::StaleModuleEndpoint,
9098 ForwardingError::UnknownReservation {
9099 client_channel: 1,
9100 module_channel: 1,
9101 },
9102 ForwardingError::ConnectionClosing {
9103 connection_id: ConnectionId::new(1),
9104 },
9105 ForwardingError::ClientEgressClosed {
9106 connection_id: ConnectionId::new(1),
9107 },
9108 ForwardingError::ModuleEgressUnavailable {
9109 connection_id: ConnectionId::new(1),
9110 },
9111 ];
9112 for err in transient {
9113 let code = forwarding_error_code(&err);
9114 assert!(
9115 client_retries(code),
9116 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
9117 );
9118 }
9119
9120 let permanent = [
9125 ForwardingError::ConnectionRoleConflict {
9126 connection_id: ConnectionId::new(1),
9127 },
9128 ForwardingError::ClientRouteChannelExhausted {
9129 connection_id: ConnectionId::new(1),
9130 },
9131 ForwardingError::ModuleRouteChannelExhausted {
9132 endpoint: ModuleEndpointId {
9133 connection_id: ConnectionId::new(1),
9134 generation: 1,
9135 },
9136 },
9137 ForwardingError::RelayCorrelationExhausted,
9138 ForwardingError::RouteOpenBuild("x".into()),
9139 ForwardingError::Poisoned,
9140 ];
9141 for err in permanent {
9142 let code = forwarding_error_code(&err);
9143 assert!(
9144 !client_retries(code),
9145 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
9146 );
9147 }
9148 }
9149
9150 #[tokio::test]
9178 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
9179 let handler = ControlHandler::default();
9180 let frame =
9181 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
9182
9183 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
9187 assert_eq!(
9188 stamped,
9189 Principal::Direct,
9190 "a caller that proved nothing must not be stamped as a supervised module"
9191 );
9192
9193 let forged = handler
9198 .route_open_principal(
9199 &frame,
9200 Some(ConsumerIdentity {
9201 module_id: "aft".to_string(),
9202 launch_nonce: "not-a-real-nonce".to_string(),
9203 }),
9204 )
9205 .unwrap();
9206 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
9207 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
9208 }
9209
9210 #[test]
9227 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
9228 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
9229 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
9230 let ClientControlRequest::RouteOpen {
9231 consumer_identity, ..
9232 } = parsed
9233 else {
9234 panic!("route.open body must parse as RouteOpen");
9235 };
9236 assert_eq!(
9237 consumer_identity,
9238 Some(ConsumerIdentity {
9239 module_id: "aft".to_string(),
9240 launch_nonce: "n".to_string(),
9241 }),
9242 "the wire field name must reach the value route_open_principal reads"
9243 );
9244 }
9245
9246 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
9247 ModuleManifest::builder(module_id, "0.1.0")
9248 .protocol_ver(protocol_ver)
9249 .provides(vec![ProviderRole::ToolProvider {
9250 tools: vec![Tool {
9251 name: "read".to_string(),
9252 description: None,
9253 execution_mode: ExecutionMode::Pure,
9254 schema: json!({"type": "object"}),
9255 }],
9256 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
9257 concurrency: Concurrency::ModuleManaged,
9258 emits_push: true,
9259 sub_supervises: true,
9260 }])
9261 .build()
9262 }
9263
9264 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
9265 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
9266 }
9267
9268 fn hello_frame_with_control_ops(
9269 module_id: &str,
9270 protocol_ver: u8,
9271 corr: u64,
9272 control_ops: Option<Vec<String>>,
9273 ) -> Frame {
9274 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
9275 }
9276
9277 fn hello_frame_with_nonce(
9278 module_id: &str,
9279 protocol_ver: u8,
9280 corr: u64,
9281 launch_nonce: Option<&str>,
9282 ) -> Frame {
9283 hello_frame_full(
9284 module_id,
9285 protocol_ver,
9286 corr,
9287 None,
9288 launch_nonce.map(ToOwned::to_owned),
9289 )
9290 }
9291
9292 fn hello_frame_full(
9293 module_id: &str,
9294 protocol_ver: u8,
9295 corr: u64,
9296 control_ops: Option<Vec<String>>,
9297 launch_nonce: Option<String>,
9298 ) -> Frame {
9299 let body = serde_json::to_vec(&ModuleHelloBody {
9300 manifest: manifest(module_id, protocol_ver),
9301 protocol_ver,
9302 control_ops,
9303 launch_nonce,
9304 })
9305 .unwrap();
9306 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9307 }
9308
9309 fn hello_frame_with_rich_ui(
9310 module_id: &str,
9311 protocol_ver: u8,
9312 corr: u64,
9313 launch_nonce: Option<&str>,
9314 rich_ui: Value,
9315 ) -> Frame {
9316 let mut body = serde_json::to_value(ModuleHelloBody {
9317 manifest: manifest(module_id, protocol_ver),
9318 protocol_ver,
9319 control_ops: None,
9320 launch_nonce: launch_nonce.map(str::to_string),
9321 })
9322 .unwrap();
9323 body["manifest"]["rich_ui"] = rich_ui;
9324 Frame::build(
9325 FrameType::Hello,
9326 control_flags(),
9327 0,
9328 0,
9329 corr,
9330 serde_json::to_vec(&body).unwrap(),
9331 )
9332 .unwrap()
9333 }
9334
9335 fn non_routable_hello_frame_with_control_ops(
9336 module_id: &str,
9337 corr: u64,
9338 control_ops: Option<Vec<String>>,
9339 ) -> Frame {
9340 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9341 manifest.provides.clear();
9342 let body = serde_json::to_vec(&ModuleHelloBody {
9343 manifest,
9344 protocol_ver: PROTOCOL_VERSION,
9345 control_ops,
9346 launch_nonce: None,
9347 })
9348 .unwrap();
9349 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9350 }
9351
9352 fn capability_grammar_hello_frame(
9353 capabilities: Value,
9354 runtime_computed: Option<Value>,
9355 corr: u64,
9356 ) -> Frame {
9357 let mut body = serde_json::to_value(ModuleHelloBody {
9358 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
9359 protocol_ver: PROTOCOL_VERSION,
9360 control_ops: None,
9361 launch_nonce: None,
9362 })
9363 .expect("HELLO body serializes");
9364 body["manifest"]["capabilities"] = capabilities;
9365 if let Some(runtime_computed) = runtime_computed {
9366 body["runtime_computed"] = runtime_computed;
9367 }
9368 Frame::build(
9369 FrameType::Hello,
9370 control_flags(),
9371 0,
9372 0,
9373 corr,
9374 serde_json::to_vec(&body).expect("HELLO body reserializes"),
9375 )
9376 .expect("HELLO frame builds")
9377 }
9378
9379 fn channel_request(channel: u16, corr: u64) -> Frame {
9380 Frame::build(
9381 FrameType::Request,
9382 Flags::new(true, Priority::Interactive, false),
9383 channel,
9384 0,
9385 corr,
9386 b"opaque".to_vec(),
9387 )
9388 .unwrap()
9389 }
9390
9391 fn route_ctx(
9392 connection_id: ConnectionId,
9393 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
9394 let (tx, rx) = mpsc::channel(8);
9395 (
9396 RouteCtx {
9397 connection_id,
9398 egress: FrameSink::new(tx),
9399 },
9400 rx,
9401 )
9402 }
9403
9404 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
9405 serde_json::from_slice(&frame.body).unwrap()
9406 }
9407
9408 async fn hello_via_sink(
9413 handler: &ControlHandler,
9414 ctx: &RouteCtx,
9415 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9416 hello: Frame,
9417 ) -> Frame {
9418 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
9419 assert!(
9420 replies.is_empty(),
9421 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
9422 );
9423 let ack = rx
9424 .try_recv()
9425 .expect("HELLO_ACK is queued on the module sink")
9426 .frame;
9427 assert_eq!(ack.header.ty, FrameType::HelloAck);
9428 ack
9429 }
9430
9431 fn parse_error(frame: &Frame) -> Value {
9432 serde_json::from_slice(&frame.body).unwrap()
9433 }
9434
9435 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
9436 serde_json::from_slice(&frame.body).unwrap()
9437 }
9438
9439 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
9440 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
9441 route_channel,
9442 route_epoch: 0,
9443 kind,
9444 })
9445 .unwrap();
9446 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9447 }
9448
9449 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
9450 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
9451 module_id: module_id.to_string(),
9452 })
9453 .unwrap();
9454 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9455 }
9456
9457 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
9458 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
9459 }
9460
9461 fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
9463 let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
9464 let mut frame = route_open_frame(corr, module_id, root);
9465 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
9466 body["identity"]["project_root"] = serde_json::json!(path.as_path());
9467 frame.body = serde_json::to_vec(&body).unwrap();
9468 frame
9469 }
9470
9471 fn route_open_frame_with_consumer_capabilities(
9472 corr: u64,
9473 module_id: &str,
9474 _project_root: ScratchDir,
9475 consumer_capabilities: Option<Vec<String>>,
9476 ) -> Frame {
9477 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9478 busy_refusals: None,
9479 target: RouteTarget::ToolProvider {
9480 module_id: module_id.to_string(),
9481 },
9482 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
9483 consumer_identity: None,
9484 consumer_capabilities,
9485 role_versions: None,
9486 admission_facts: None,
9487 scope: None,
9488 })
9489 .unwrap();
9490 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9491 }
9492
9493 fn route_open_frame_with_role_versions(
9494 corr: u64,
9495 module_id: &str,
9496 _project_root: ScratchDir,
9497 role_versions: Option<BTreeMap<String, String>>,
9498 ) -> Frame {
9499 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9500 busy_refusals: None,
9501 target: RouteTarget::ToolProvider {
9502 module_id: module_id.to_string(),
9503 },
9504 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9505 consumer_identity: None,
9506 consumer_capabilities: None,
9507 role_versions,
9508 admission_facts: None,
9509 scope: None,
9510 })
9511 .unwrap();
9512 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9513 }
9514
9515 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
9516 entries
9517 .iter()
9518 .map(|(role, version)| (role.to_string(), version.to_string()))
9519 .collect()
9520 }
9521
9522 fn route_open_frame_with_admission_facts(
9523 corr: u64,
9524 module_id: &str,
9525 _project_root: ScratchDir,
9526 consumer_identity: Option<subc_control::ConsumerIdentity>,
9527 facts: Option<Value>,
9528 ) -> Frame {
9529 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9530 busy_refusals: None,
9531 target: RouteTarget::ToolProvider {
9532 module_id: module_id.to_string(),
9533 },
9534 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9535 consumer_identity,
9536 consumer_capabilities: None,
9537 role_versions: None,
9538 admission_facts: facts,
9539 scope: None,
9540 })
9541 .unwrap();
9542 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9543 }
9544
9545 #[derive(Clone, Default)]
9546 struct EventCapture {
9547 events: Arc<Mutex<Vec<CapturedEvent>>>,
9548 }
9549
9550 #[derive(Clone, Debug)]
9551 struct CapturedEvent {
9552 target: String,
9553 level: tracing::Level,
9554 fields: BTreeMap<String, String>,
9555 }
9556
9557 impl EventCapture {
9558 fn events(&self) -> Vec<CapturedEvent> {
9559 self.events.lock().unwrap().clone()
9560 }
9561 }
9562
9563 impl<S> Layer<S> for EventCapture
9564 where
9565 S: Subscriber,
9566 {
9567 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
9568 let mut visitor = EventFieldVisitor::default();
9569 event.record(&mut visitor);
9570 self.events.lock().unwrap().push(CapturedEvent {
9571 target: event.metadata().target().to_string(),
9572 level: *event.metadata().level(),
9573 fields: visitor.fields,
9574 });
9575 }
9576 }
9577
9578 #[derive(Default)]
9579 struct EventFieldVisitor {
9580 fields: BTreeMap<String, String>,
9581 }
9582
9583 impl Visit for EventFieldVisitor {
9584 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
9585 self.fields
9586 .insert(field.name().to_string(), format!("{value:?}"));
9587 }
9588 }
9589
9590 fn health_response(corr: u64, status: HealthStatus) -> Frame {
9591 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
9592 status,
9593 detail: Some("warming".to_string()),
9594 metrics: Some(json!({"queue_depth": 3})),
9595 })
9596 .unwrap();
9597 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9598 }
9599
9600 fn route_bind_ack(corr: u64) -> Frame {
9601 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
9602 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9603 }
9604
9605 fn unique_project_root(label: &str) -> ScratchDir {
9606 ScratchDir::new(label)
9607 }
9608
9609 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
9610 match parse_route_poll(frame) {
9611 ClientControlResponse::RoutePoll {
9612 status: None,
9613 live: Some(live),
9614 ..
9615 } => assert_eq!(live, expected_live),
9616 other => panic!("unexpected route.poll response: {other:?}"),
9617 }
9618 }
9619
9620 fn bind_liveness_route(
9621 registry: &Registry,
9622 forwarding: &ForwardingTable,
9623 module_id: &str,
9624 ) -> (RouteCtx, u16, u32) {
9625 let module_connection = ConnectionId::new(101);
9626 let client_connection = ConnectionId::new(202);
9627 let registration = registry
9628 .register_with_control_ops(
9629 manifest(module_id, PROTOCOL_VERSION),
9630 PROTOCOL_VERSION,
9631 module_connection,
9632 module_baseline_control_ops(),
9633 )
9634 .unwrap();
9635 let (module_tx, _module_rx) = mpsc::channel(8);
9636 let endpoint = forwarding
9637 .register_module_connection(
9638 module_connection,
9639 module_id.to_string(),
9640 PROTOCOL_VERSION,
9641 manifest_concurrency(®istration.manifest),
9642 FrameSink::new(module_tx),
9643 )
9644 .unwrap();
9645 let (client_ctx, _client_rx) = route_ctx(client_connection);
9646 let pending = forwarding
9647 .begin_route_bind_relay_for_test(
9648 client_connection,
9649 client_ctx.egress.clone(),
9650 1,
9651 module_id,
9652 )
9653 .unwrap();
9654 assert_eq!(pending.endpoint, endpoint);
9655 let route_channel = pending.client_channel;
9656 let route_epoch = pending.client_epoch;
9657 forwarding
9658 .complete_pending_relay(
9659 module_connection,
9660 pending.corr,
9661 RouteBindRelayOutcome::Accepted,
9662 )
9663 .unwrap();
9664 (client_ctx, route_channel, route_epoch)
9665 }
9666
9667 struct FakeProcessLiveness {
9668 live: Option<bool>,
9669 }
9670
9671 impl ModuleProcessLiveness for FakeProcessLiveness {
9672 fn process_live(&self, _module_id: &str) -> Option<bool> {
9673 self.live
9674 }
9675 }
9676
9677 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9678 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9679 {
9680 let registry = Arc::new(Registry::default());
9681 let supervisor_handle = SupervisorHandle::new();
9682 let supervisor = Supervisor::new_for_test(
9683 Arc::clone(®istry),
9684 RestartPolicy::new(1, Duration::from_millis(10)),
9685 )
9686 .with_handle(supervisor_handle.clone());
9687 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9688 let module = supervisor
9689 .spawn(ModuleSpec {
9690 carry_containment: false,
9691 module_id: "stderr-tail-wire".to_string(),
9692 program: fake_aft_stub_path(),
9693 args: Vec::new(),
9694 env: vec![
9695 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9696 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9697 ],
9698 reserved: false,
9699 reserved_prefixes: Vec::new(),
9700 protocol: ModuleProtocol::Subc,
9701 overlap: Default::default(),
9702 })
9703 .unwrap();
9704
9705 let deadline = Instant::now() + Duration::from_secs(5);
9706 loop {
9707 let tail = module.stderr_tail(None, None);
9708 if tail
9709 .entries
9710 .iter()
9711 .any(|entry| matches!(entry, TailEntry::ProcessStart))
9712 && tail.entries.iter().any(|entry| {
9713 matches!(
9714 entry,
9715 TailEntry::Line {
9716 truncated: true,
9717 ..
9718 }
9719 )
9720 })
9721 {
9722 break;
9723 }
9724 assert!(
9725 Instant::now() < deadline,
9726 "module did not produce a truncated line and restart boundary: {tail:?}"
9727 );
9728 sleep(Duration::from_millis(10)).await;
9729 }
9730
9731 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9732 let request = ClientControlRequest::SupervisorStderrTail {
9733 module_id: "stderr-tail-wire".to_string(),
9734 max_lines: None,
9735 max_bytes: None,
9736 };
9737 let frame = Frame::build(
9738 FrameType::Request,
9739 control_flags(),
9740 0,
9741 0,
9742 1,
9743 serde_json::to_vec(&request).unwrap(),
9744 )
9745 .unwrap();
9746 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9747 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9748 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9749 serde_json::from_slice(&responses[0].body).unwrap()
9750 else {
9751 panic!("expected supervisor.stderr_tail response");
9752 };
9753
9754 assert!(
9755 tail.entries
9756 .iter()
9757 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9758 "the control response lost the restart boundary"
9759 );
9760 let Some(StderrTailEntry::Line {
9761 text,
9762 truncated,
9763 at_ms,
9764 }) = tail.entries.iter().find(|entry| {
9765 matches!(
9766 entry,
9767 StderrTailEntry::Line {
9768 truncated: true,
9769 ..
9770 }
9771 )
9772 })
9773 else {
9774 panic!("the control response lost the truncated line");
9775 };
9776 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9777 assert!(*truncated);
9778 assert!(
9779 at_ms.is_some(),
9780 "the control response lost the line's capture time"
9781 );
9782 }
9783
9784 #[tokio::test(flavor = "current_thread")]
9789 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9790 let dir = ScratchDir::new("terminals-off-worker");
9791 let journal_path = dir.join("terminals.jsonl");
9792 let registry = Arc::new(Registry::default());
9793 let supervisor_handle = SupervisorHandle::new();
9794 let supervisor =
9795 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9796 .with_handle(supervisor_handle.clone())
9797 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9798 let module = supervisor
9799 .spawn(ModuleSpec {
9800 carry_containment: false,
9801 module_id: "terminal-off-worker".to_string(),
9802 program: fake_aft_stub_path(),
9803 args: Vec::new(),
9804 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9805 reserved: false,
9806 reserved_prefixes: Vec::new(),
9807 protocol: ModuleProtocol::Subc,
9808 overlap: Default::default(),
9809 })
9810 .unwrap();
9811 let deadline = Instant::now() + Duration::from_secs(5);
9812 while module.terminal_history().entries.len() != 2 {
9813 assert!(Instant::now() < deadline, "module did not record two exits");
9814 sleep(Duration::from_millis(10)).await;
9815 }
9816
9817 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9818 let handler =
9819 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
9820 let frame = Frame::build(
9821 FrameType::Request,
9822 control_flags(),
9823 0,
9824 0,
9825 1,
9826 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9827 module_id: "terminal-off-worker".to_string(),
9828 })
9829 .unwrap(),
9830 )
9831 .unwrap();
9832 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9833 let spawned_at = std::time::Instant::now();
9834 let read = tokio::spawn({
9835 let handler = Arc::clone(&handler);
9836 async move { handler.handle_control_frame(&ctx, frame).await }
9837 });
9838 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9841 .await
9842 .unwrap()
9843 .expect("the history read reached its pause");
9844 let elapsed = spawned_at.elapsed();
9845 assert!(
9846 elapsed < Duration::from_secs(2) && !read.is_finished(),
9847 "this task could not run while the history read was paused \
9848 (resumed after {elapsed:?}, read finished: {})",
9849 read.is_finished()
9850 );
9851
9852 drop(release);
9853 let responses = read.await.unwrap().unwrap();
9854 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9855 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9856 panic!("expected supervisor.terminals response");
9857 };
9858 assert_eq!(terminals.entries.len(), 2);
9859 assert_eq!(terminals.journal_skipped_lines, 0);
9860 assert_eq!(terminals.journal_read_errors, 0);
9861 }
9862
9863 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9864 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9865 let registry = Arc::new(Registry::default());
9866 let supervisor_handle = SupervisorHandle::new();
9867 let supervisor =
9868 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9869 .with_handle(supervisor_handle.clone());
9870 let module = supervisor
9871 .spawn(ModuleSpec {
9872 carry_containment: false,
9873 module_id: "terminal-golden".to_string(),
9874 program: fake_aft_stub_path(),
9875 args: Vec::new(),
9876 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9877 reserved: false,
9878 reserved_prefixes: Vec::new(),
9879 protocol: ModuleProtocol::Subc,
9880 overlap: Default::default(),
9881 })
9882 .unwrap();
9883
9884 let deadline = Instant::now() + Duration::from_secs(5);
9885 while module.terminal_history().entries.len() != 2 {
9886 assert!(
9887 Instant::now() < deadline,
9888 "module did not retain two terminal exits: {:?}",
9889 module.terminal_history()
9890 );
9891 sleep(Duration::from_millis(10)).await;
9892 }
9893
9894 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9895 let request = ClientControlRequest::SupervisorTerminals {
9896 module_id: "terminal-golden".to_string(),
9897 };
9898 let frame = Frame::build(
9899 FrameType::Request,
9900 control_flags(),
9901 0,
9902 0,
9903 1,
9904 serde_json::to_vec(&request).unwrap(),
9905 )
9906 .unwrap();
9907 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9908 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9909 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9910 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9911 panic!("expected supervisor.terminals response");
9912 };
9913 assert_eq!(terminals.entries.len(), 2);
9914 assert_eq!(terminals.dropped, 0);
9915
9916 let mut rendered = serde_json::to_value(response).unwrap();
9917 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9920 for (index, entry) in rendered["entries"]
9921 .as_array_mut()
9922 .expect("terminal response entries array")
9923 .iter_mut()
9924 .enumerate()
9925 {
9926 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9927 }
9928
9929 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9930 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9931 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9932 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9933 std::fs::write(&golden_path, &serialized).unwrap();
9934 }
9935 let expected: Value =
9936 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9937 assert_eq!(rendered, expected);
9938 }
9939
9940 #[test]
9941 fn hello_registers_manifest_and_returns_ack() {
9942 let registry = Arc::new(Registry::default());
9943 let handler = ControlHandler::new(Arc::clone(®istry));
9944 let conn = ConnectionId::new(1);
9945
9946 let responses = handler
9947 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9948 .unwrap();
9949
9950 assert_eq!(responses.len(), 1);
9951 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9952 assert_eq!(responses[0].header.channel, 0);
9953 assert_eq!(responses[0].header.corr, 7);
9954 let ack = parse_ack(&responses[0]);
9955 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9956 assert!(ack
9957 .subc_capabilities
9958 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9959 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9960 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9961 assert!(ack
9962 .subc_ops
9963 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9964 assert!(ack
9965 .subc_ops
9966 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9967
9968 let registration = registry.get_module("aft").unwrap().unwrap();
9969 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9970 assert_eq!(registration.state, ChannelState::Active);
9971 assert_eq!(registration.connection_id, conn);
9972 assert_eq!(registration.control_ops, module_baseline_control_ops());
9973 assert!(registration.manifest.display_name.is_none());
9974 }
9975
9976 #[test]
9977 fn hello_display_name_refuses_invalid_manifest_by_field() {
9978 let registry = Arc::new(Registry::default());
9979 let handler = ControlHandler::new(Arc::clone(®istry));
9980 for (index, invalid) in [
9981 json!(""),
9982 json!("a".repeat(41)),
9983 json!(" Routing"),
9984 json!("a\nb"),
9985 json!("a\u{202e}b"),
9986 json!(7),
9987 ]
9988 .into_iter()
9989 .enumerate()
9990 {
9991 let module_id = format!("bad-display-name-{index}");
9992 let mut body = serde_json::to_value(ModuleHelloBody {
9993 manifest: manifest(&module_id, PROTOCOL_VERSION),
9994 protocol_ver: PROTOCOL_VERSION,
9995 control_ops: None,
9996 launch_nonce: None,
9997 })
9998 .unwrap();
9999 body["manifest"]["display_name"] = invalid;
10000 let hello = Frame::build(
10001 FrameType::Hello,
10002 control_flags(),
10003 0,
10004 0,
10005 7,
10006 serde_json::to_vec(&body).unwrap(),
10007 )
10008 .unwrap();
10009 let response = handler
10010 .handle_control(ConnectionId::new(100 + index as u64), hello)
10011 .unwrap();
10012 assert_eq!(response[0].header.ty, FrameType::Error);
10013 let error = parse_error(&response[0]);
10014 assert_eq!(error["code"], "invalid_manifest");
10015 assert!(error["message"].as_str().unwrap().contains("display_name"));
10016 assert!(registry.get_module(&module_id).unwrap().is_none());
10017 }
10018 }
10019
10020 #[test]
10021 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
10022 let invalid_identifiers = [
10023 ("case_change", "credentials-Provider/v1"),
10024 ("leading_zero", "credentials-provider/v01"),
10025 ("trailing_hyphen", "credentials-provider-/v1"),
10026 ("consecutive_hyphens", "credentials--provider/v1"),
10027 ("uppercase", "Credentials-provider/v1"),
10028 ("missing_v", "credentials-provider/1"),
10029 ("whitespace", "credentials provider/v1"),
10030 ("zero_version", "credentials-provider/v0"),
10031 ("out_of_range_version", "credentials-provider/v4294967296"),
10032 (
10033 "overlength_name",
10034 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
10035 ),
10036 ];
10037 let mut cases = invalid_identifiers
10038 .into_iter()
10039 .map(|(name, identifier)| {
10040 (
10041 format!("identifier_{name}"),
10042 "capabilities.provides[0]".to_string(),
10043 identifier.to_string(),
10044 json!({ "provides": [identifier] }),
10045 None,
10046 )
10047 })
10048 .collect::<Vec<_>>();
10049 cases.extend([
10050 (
10051 "unknown_need".to_string(),
10052 "capabilities.requires[0].need".to_string(),
10053 "deferred".to_string(),
10054 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
10055 None,
10056 ),
10057 (
10058 "duplicate_provides".to_string(),
10059 "capabilities.provides[1]".to_string(),
10060 "credentials-provider/v1".to_string(),
10061 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
10062 None,
10063 ),
10064 (
10065 "duplicate_must_never_reach".to_string(),
10066 "capabilities.must_never_reach[1]".to_string(),
10067 "credentials-provider/v1".to_string(),
10068 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
10069 None,
10070 ),
10071 (
10072 "duplicate_requires_same_need".to_string(),
10073 "capabilities.requires[1]".to_string(),
10074 "credentials-provider/v1".to_string(),
10075 json!({ "requires": [
10076 { "capability": "credentials-provider/v1", "need": "required" },
10077 { "capability": "credentials-provider/v1", "need": "required" }
10078 ] }),
10079 None,
10080 ),
10081 (
10082 "duplicate_requires_conflicting_need".to_string(),
10083 "capabilities.requires[1]".to_string(),
10084 "credentials-provider/v1".to_string(),
10085 json!({ "requires": [
10086 { "capability": "credentials-provider/v1", "need": "required" },
10087 { "capability": "credentials-provider/v1", "need": "optional" }
10088 ] }),
10089 None,
10090 ),
10091 (
10092 "capabilities_root_pointer".to_string(),
10093 "runtime_computed[0]".to_string(),
10094 "/capabilities".to_string(),
10095 json!({}),
10096 Some(json!(["/capabilities"])),
10097 ),
10098 (
10099 "capabilities_descendant_pointer".to_string(),
10100 "runtime_computed[0]".to_string(),
10101 "/capabilities/provides".to_string(),
10102 json!({}),
10103 Some(json!(["/capabilities/provides"])),
10104 ),
10105 (
10106 "malformed_pointer_without_leading_slash".to_string(),
10107 "runtime_computed[0]".to_string(),
10108 "capabilities".to_string(),
10109 json!({}),
10110 Some(json!(["capabilities"])),
10111 ),
10112 (
10113 "malformed_pointer_escape".to_string(),
10114 "runtime_computed[0]".to_string(),
10115 "/roles/~2/tools".to_string(),
10116 json!({}),
10117 Some(json!(["/roles/~2/tools"])),
10118 ),
10119 (
10120 "unknown_capabilities_field".to_string(),
10121 "capabilities.future".to_string(),
10122 "<array>".to_string(),
10123 json!({ "future": [] }),
10124 None,
10125 ),
10126 ]);
10127
10128 for (index, (name, field, value, capabilities, runtime_computed)) in
10129 cases.into_iter().enumerate()
10130 {
10131 let registry = Arc::new(Registry::default());
10132 let handler = ControlHandler::new(Arc::clone(®istry));
10133 let response = handler
10134 .handle_control(
10135 ConnectionId::new((index + 1) as u64),
10136 capability_grammar_hello_frame(
10137 capabilities,
10138 runtime_computed,
10139 index as u64 + 1,
10140 ),
10141 )
10142 .expect("invalid HELLO returns a refusal");
10143
10144 assert_eq!(response.len(), 1, "{name} must emit one refusal");
10145 let error = parse_error(&response[0]);
10146 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
10147 let message = error["message"]
10148 .as_str()
10149 .expect("error message is a string");
10150 assert!(
10151 message.contains(&field),
10152 "{name}: field missing from {message}"
10153 );
10154 assert!(
10155 message.contains(&value),
10156 "{name}: value missing from {message}"
10157 );
10158 assert_eq!(
10159 registry
10160 .active_registration_count()
10161 .expect("registry reads"),
10162 0,
10163 "{name}: refused HELLO must not create a catalog entry"
10164 );
10165 }
10166 }
10167
10168 #[test]
10169 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
10170 let registry = Arc::new(Registry::default());
10171 let handler = ControlHandler::new(Arc::clone(®istry));
10172 let capabilities = json!({
10173 "provides": ["credentials-provider/v1"],
10174 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
10175 "must_never_reach": ["federation-transport/v1"]
10176 });
10177 let response = handler
10178 .handle_control(
10179 ConnectionId::new(99),
10180 capability_grammar_hello_frame(
10181 capabilities.clone(),
10182 Some(json!(["/roles/0/tools"])),
10183 99,
10184 ),
10185 )
10186 .expect("valid HELLO registers");
10187 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10188
10189 let request = Frame::build(
10190 FrameType::Request,
10191 control_flags(),
10192 0,
10193 0,
10194 100,
10195 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10196 .expect("catalog request serializes"),
10197 )
10198 .expect("catalog request frame builds");
10199 let response = handler
10200 .handle_catalog_list(request, None)
10201 .expect("catalog list succeeds");
10202 let ClientControlResponse::CatalogList { modules, .. } =
10203 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
10204 else {
10205 panic!("catalog request must return catalog.list");
10206 };
10207 assert_eq!(modules.len(), 1);
10208 assert_eq!(
10209 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
10210 capabilities
10211 );
10212 }
10213
10214 #[test]
10215 fn catalog_list_mirrors_management_operation_description() {
10216 let registry = Arc::new(Registry::default());
10217 let handler = ControlHandler::new(Arc::clone(®istry));
10218 let description = "List managed records and return their identifiers and metadata.";
10219 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
10220 manifest.provides = vec![ProviderRole::ManagementSurface {
10221 operations: vec![ManagementOperation {
10222 name: "records.list".to_string(),
10223 kind: ManagementOperationKind::Query,
10224 description: Some(description.to_string()),
10225 }],
10226 config_schema: json!({"type": "object"}),
10227 observability: vec![ObservabilitySurface {
10228 name: "records.stats".to_string(),
10229 kind: ObservabilityKind::Snapshot,
10230 }],
10231 identity_scope: vec![IdentityScope::Project],
10232 concurrency: Concurrency::ModuleManaged,
10233 }];
10234 registry
10235 .register_with_control_ops(
10236 manifest,
10237 PROTOCOL_VERSION,
10238 ConnectionId::new(99),
10239 Vec::new(),
10240 )
10241 .expect("described management manifest registers");
10242
10243 let request = Frame::build(
10244 FrameType::Request,
10245 control_flags(),
10246 0,
10247 0,
10248 100,
10249 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10250 .expect("catalog request serializes"),
10251 )
10252 .expect("catalog request frame builds");
10253 let response = handler
10254 .handle_catalog_list(request, None)
10255 .expect("catalog list succeeds");
10256 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10257 assert_eq!(
10258 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
10259 "catalog.list must preserve the declared operation description verbatim"
10260 );
10261 }
10262
10263 #[test]
10264 fn catalog_list_mirrors_registered_operation_delivery() {
10265 let registry = Arc::new(Registry::default());
10266 let handler = ControlHandler::new(Arc::clone(®istry));
10267 let delivery =
10268 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
10269 let mut body = serde_json::to_value(ModuleHelloBody {
10270 manifest: manifest("watch-management", PROTOCOL_VERSION),
10271 protocol_ver: PROTOCOL_VERSION,
10272 control_ops: None,
10273 launch_nonce: None,
10274 })
10275 .unwrap();
10276 body["manifest"]["provides"] = json!([{
10277 "role": "management_surface",
10278 "operations": [
10279 {"name": "status", "kind": "query"},
10280 {"name": "future", "kind": "query"},
10281 {"name": "reset", "kind": "mutate"}
10282 ],
10283 "config_schema": {}, "observability": [], "identity_scope": []
10284 }]);
10285 body["manifest"]["operation_delivery"] = delivery.clone();
10286 let hello = Frame::build(
10287 FrameType::Hello,
10288 control_flags(),
10289 0,
10290 0,
10291 99,
10292 serde_json::to_vec(&body).unwrap(),
10293 )
10294 .unwrap();
10295 let response = handler
10296 .handle_control(ConnectionId::new(99), hello)
10297 .unwrap();
10298 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10299
10300 let (_, registrations) = registry.list_modules().unwrap();
10301 let registered = ®istrations[0].manifest;
10302 assert!(registered.is_watch_operation("status"));
10303 assert!(!registered.is_watch_operation("future"));
10304 assert!(!registered.is_watch_operation("reset"));
10305 assert!(!registered.is_watch_operation("missing"));
10306
10307 let request = Frame::build(
10308 FrameType::Request,
10309 control_flags(),
10310 0,
10311 0,
10312 100,
10313 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10314 )
10315 .unwrap();
10316 let response = handler.handle_catalog_list(request, None).unwrap();
10317 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
10318 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
10319 let ClientControlResponse::CatalogList { modules, .. } =
10320 serde_json::from_slice(&response[0].body).unwrap()
10321 else {
10322 panic!("expected catalog.list");
10323 };
10324 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
10325 }
10326
10327 #[test]
10328 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
10329 let registry = Arc::new(Registry::default());
10330 let supervisor = SupervisorHandle::new();
10331 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10332 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10333 let descriptor = json!({
10334 "schema": "ckios.richui.manifest/1",
10335 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10336 "bytes": 4096,
10337 "chunks": 4
10338 });
10339 let connection = ConnectionId::new(201);
10340 let hello = handler
10341 .handle_control(
10342 connection,
10343 hello_frame_with_rich_ui(
10344 "vault",
10345 PROTOCOL_VERSION,
10346 201,
10347 Some("the-real-nonce"),
10348 descriptor.clone(),
10349 ),
10350 )
10351 .unwrap();
10352 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10353
10354 let request =
10355 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
10356 let provides = registry
10357 .get_module_by_connection(connection)
10358 .unwrap()
10359 .unwrap()
10360 .manifest
10361 .provides;
10362 handler
10363 .handle_catalog_update(connection, request, provides, None, Some(false))
10364 .unwrap();
10365
10366 let request = Frame::build(
10367 FrameType::Request,
10368 control_flags(),
10369 0,
10370 0,
10371 203,
10372 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10373 )
10374 .unwrap();
10375 let response = handler.handle_catalog_list(request, None).unwrap();
10376 let ClientControlResponse::CatalogList { modules, .. } =
10377 serde_json::from_slice(&response[0].body).unwrap()
10378 else {
10379 panic!("expected catalog.list");
10380 };
10381 assert_eq!(
10382 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
10383 descriptor
10384 );
10385 assert!(modules[0].nonce_verified);
10386 }
10387
10388 fn hello_with_role_declarations(declarations: Value) -> Frame {
10389 let mut body = serde_json::to_value(ModuleHelloBody {
10390 manifest: manifest("role-test", PROTOCOL_VERSION),
10391 protocol_ver: PROTOCOL_VERSION,
10392 control_ops: None,
10393 launch_nonce: None,
10394 })
10395 .unwrap();
10396 body["manifest"]["role_declarations"] = declarations;
10397 Frame::build(
10398 FrameType::Hello,
10399 control_flags(),
10400 0,
10401 0,
10402 301,
10403 serde_json::to_vec(&body).unwrap(),
10404 )
10405 .unwrap()
10406 }
10407
10408 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
10409 let request = Frame::build(
10410 FrameType::Request,
10411 control_flags(),
10412 0,
10413 0,
10414 302,
10415 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10416 )
10417 .unwrap();
10418 let response = handler.handle_catalog_list(request, None).unwrap();
10419 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
10420 let ClientControlResponse::CatalogList { mut modules, .. } =
10421 serde_json::from_slice(&response[0].body).unwrap()
10422 else {
10423 panic!("expected catalog.list");
10424 };
10425 (raw["modules"][0].clone(), modules.remove(0))
10426 }
10427
10428 #[test]
10429 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
10430 let registry = Arc::new(Registry::default());
10431 let handler = ControlHandler::new(Arc::clone(®istry));
10432 let declarations = json!({
10433 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
10434 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
10435 });
10436 let connection = ConnectionId::new(301);
10437 let response = handler
10438 .handle_control(
10439 connection,
10440 hello_with_role_declarations(declarations.clone()),
10441 )
10442 .unwrap();
10443 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10444 let registered = registry
10445 .get_module_by_connection(connection)
10446 .unwrap()
10447 .unwrap()
10448 .manifest;
10449 assert_eq!(
10453 serde_json::to_value(®istered.role_declarations).unwrap(),
10454 declarations
10455 );
10456 let (raw, entry) = role_declarations_catalog(&handler);
10457 assert_eq!(raw["role_declarations"], declarations);
10458 assert_eq!(entry.role_declarations, registered.role_declarations);
10459
10460 let request =
10461 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
10462 handler
10463 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
10464 .unwrap();
10465 let (raw, entry) = role_declarations_catalog(&handler);
10466 assert_eq!(raw["role_declarations"], declarations);
10467 assert_eq!(entry.role_declarations, registered.role_declarations);
10468 }
10469
10470 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
10471 let registry = Arc::new(Registry::default());
10472 let handler = ControlHandler::new(Arc::clone(®istry));
10473 let capture = EventCapture::default();
10474 let _subscriber =
10475 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10476 let response = handler
10477 .handle_control(
10478 ConnectionId::new(301),
10479 hello_with_role_declarations(declarations),
10480 )
10481 .unwrap();
10482 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10483 assert!(registry
10484 .get_module_by_connection(ConnectionId::new(301))
10485 .unwrap()
10486 .unwrap()
10487 .manifest
10488 .role_declarations
10489 .is_empty());
10490 let (raw, entry) = role_declarations_catalog(&handler);
10491 assert!(raw.get("role_declarations").is_none());
10492 assert!(entry.role_declarations.is_empty());
10493 let warnings: Vec<_> = capture
10494 .events()
10495 .into_iter()
10496 .filter(|event| {
10497 event.fields.get("message").map(String::as_str)
10498 == Some("dropping whole role_declarations map from HELLO")
10499 })
10500 .collect();
10501 assert_eq!(warnings.len(), 1);
10502 assert_eq!(warnings[0].level, tracing::Level::WARN);
10503 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
10504 assert_eq!(warnings[0].fields["reason"], reason);
10505 }
10506
10507 #[test]
10508 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
10509 for value in [
10510 json!(null),
10511 json!(false),
10512 json!(0),
10513 json!("opaque"),
10514 json!([]),
10515 ] {
10516 assert_hello_drops_role_declarations_with_warning(
10517 json!({"routing/v1": {}, "broken/v1": value}),
10518 "role_declarations values must be objects",
10519 );
10520 }
10521 assert_hello_drops_role_declarations_with_warning(
10522 json!([]),
10523 "role_declarations must be an object",
10524 );
10525 }
10526
10527 #[test]
10528 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
10529 assert_hello_drops_role_declarations_with_warning(
10530 json!({"routing/v1": {}, "bad key": {}}),
10531 "role_declarations key must use capability-name grammar",
10532 );
10533 }
10534
10535 #[test]
10536 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
10537 assert_hello_drops_role_declarations_with_warning(
10538 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
10539 "role_declarations exceeds 16384 compact JSON bytes",
10540 );
10541 }
10542
10543 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
10544 let mut body = serde_json::to_value(ModuleHelloBody {
10545 manifest: manifest("flow-ops", PROTOCOL_VERSION),
10546 protocol_ver: PROTOCOL_VERSION,
10547 control_ops: None,
10548 launch_nonce: None,
10549 })
10550 .unwrap();
10551 body["manifest"]["provides"] = json!([{
10552 "role": "management_surface",
10553 "operations": [
10554 {"name": "diagnostics.findings", "kind": "query"},
10555 {"name": "reset", "kind": "mutate"}
10556 ],
10557 "config_schema": {}, "observability": [], "identity_scope": []
10558 }]);
10559 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
10560 if let Some(flow_operations) = flow_operations {
10561 body["manifest"]["flow_operations"] = flow_operations;
10562 }
10563 Frame::build(
10564 FrameType::Hello,
10565 control_flags(),
10566 0,
10567 0,
10568 401,
10569 serde_json::to_vec(&body).unwrap(),
10570 )
10571 .unwrap()
10572 }
10573
10574 #[test]
10575 fn catalog_list_mirrors_flow_operations_unchanged() {
10576 for (capability, flow_operations) in [
10579 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
10580 ("call-modes/v1", json!(["diagnostics.findings"])),
10581 ("flow-scopes/v1", json!([])),
10582 ] {
10583 let registry = Arc::new(Registry::default());
10584 let handler = ControlHandler::new(Arc::clone(®istry));
10585 let connection = ConnectionId::new(401);
10586 let response = handler
10587 .handle_control(
10588 connection,
10589 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
10590 )
10591 .unwrap();
10592 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10593 let registered = registry
10594 .get_module_by_connection(connection)
10595 .unwrap()
10596 .unwrap()
10597 .manifest;
10598 assert_eq!(
10599 serde_json::to_value(®istered.flow_operations).unwrap(),
10600 flow_operations
10601 );
10602 let (raw, entry) = role_declarations_catalog(&handler);
10603 assert_eq!(raw["flow_operations"], flow_operations);
10604 assert_eq!(entry.flow_operations, registered.flow_operations);
10605 }
10606
10607 let registry = Arc::new(Registry::default());
10608 let handler = ControlHandler::new(Arc::clone(®istry));
10609 let response = handler
10610 .handle_control(
10611 ConnectionId::new(401),
10612 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
10613 )
10614 .unwrap();
10615 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10616 let (raw, entry) = role_declarations_catalog(&handler);
10617 assert!(raw.get("flow_operations").is_none());
10618 assert_eq!(entry.flow_operations, None);
10619 }
10620
10621 fn assert_hello_drops_flow_operations_with_warning(
10622 capabilities: Value,
10623 flow_operations: Value,
10624 reason: &str,
10625 ) {
10626 let registry = Arc::new(Registry::default());
10627 let handler = ControlHandler::new(Arc::clone(®istry));
10628 let capture = EventCapture::default();
10629 let _subscriber =
10630 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10631 let response = handler
10632 .handle_control(
10633 ConnectionId::new(401),
10634 hello_with_flow_operations(capabilities, Some(flow_operations)),
10635 )
10636 .unwrap();
10637 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10638 let registered = registry
10639 .get_module_by_connection(ConnectionId::new(401))
10640 .unwrap()
10641 .expect("the module still registers")
10642 .manifest;
10643 assert_eq!(registered.module_id, "flow-ops");
10644 assert_eq!(registered.flow_operations, None);
10645 let (raw, entry) = role_declarations_catalog(&handler);
10646 assert_eq!(raw["module_id"], "flow-ops");
10647 assert!(raw.get("flow_operations").is_none());
10648 assert_eq!(entry.flow_operations, None);
10649 let warnings: Vec<_> = capture
10650 .events()
10651 .into_iter()
10652 .filter(|event| {
10653 event.fields.get("message").map(String::as_str)
10654 == Some("dropping flow_operations from HELLO")
10655 })
10656 .collect();
10657 assert_eq!(warnings.len(), 1);
10658 assert_eq!(warnings[0].level, tracing::Level::WARN);
10659 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
10660 assert_eq!(warnings[0].fields["reason"], reason);
10661 }
10662
10663 #[test]
10664 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
10665 assert_hello_drops_flow_operations_with_warning(
10666 json!(["flow-scopes/v1"]),
10667 json!(["diagnostics.findings", "missing"]),
10668 "flow_operations names 'missing', which is not a declared management operation",
10669 );
10670 }
10671
10672 #[test]
10673 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
10674 assert_hello_drops_flow_operations_with_warning(
10675 json!([]),
10676 json!(["diagnostics.findings"]),
10677 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10678 );
10679 assert_hello_drops_flow_operations_with_warning(
10681 json!(["agent-run-scopes/v1"]),
10682 json!([]),
10683 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10684 );
10685 }
10686
10687 #[test]
10688 fn hello_malformed_flow_operations_is_dropped_with_warning() {
10689 assert_hello_drops_flow_operations_with_warning(
10690 json!(["flow-scopes/v1"]),
10691 json!(["reset", "reset"]),
10692 "flow_operations names 'reset' more than once",
10693 );
10694 assert_hello_drops_flow_operations_with_warning(
10695 json!(["flow-scopes/v1"]),
10696 json!(vec!["reset"; 257]),
10697 "flow_operations lists 257 names, more than 256",
10698 );
10699 assert_hello_drops_flow_operations_with_warning(
10700 json!(["flow-scopes/v1"]),
10701 json!("reset"),
10702 "flow_operations must be an array of operation names",
10703 );
10704 }
10705
10706 #[test]
10707 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
10708 let registry = Arc::new(Registry::default());
10709 let handler = ControlHandler::new(registry.clone());
10710 let descriptor = json!({
10711 "schema": "ckios.richui.manifest/1",
10712 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10713 "bytes": 128,
10714 "chunks": 2
10715 });
10716 let malformed = [
10717 ("schema", json!("")),
10718 ("schema", json!("a".repeat(65))),
10719 ("schema", json!("Bad schema")),
10720 (
10721 "manifest_digest",
10722 json!(format!("sha256:{}", "A".repeat(64))),
10723 ),
10724 ("manifest_digest", json!("sha256:abc")),
10725 ("bytes", json!(0)),
10726 ("bytes", json!(16_777_217)),
10727 ("bytes", json!("128")),
10728 ("chunks", json!(0)),
10729 ("chunks", json!(1025)),
10730 ("chunks", json!("2")),
10731 ];
10732 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
10733 let mut invalid = descriptor.clone();
10734 invalid[field] = invalid_value;
10735 let response = handler
10736 .handle_control(
10737 ConnectionId::new(210 + index as u64),
10738 hello_frame_with_rich_ui(
10739 &format!("bad-rich-ui-{index}"),
10740 PROTOCOL_VERSION,
10741 210 + index as u64,
10742 None,
10743 invalid,
10744 ),
10745 )
10746 .unwrap();
10747 assert_eq!(response[0].header.ty, FrameType::Error);
10748 let error = parse_error(&response[0]);
10749 assert_eq!(error["code"], "invalid_manifest");
10750 assert!(error["message"]
10751 .as_str()
10752 .unwrap()
10753 .contains(&format!("rich_ui.{field}")));
10754 assert!(registry
10755 .get_module(&format!("bad-rich-ui-{index}"))
10756 .unwrap()
10757 .is_none());
10758 }
10759 }
10760
10761 #[test]
10762 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
10763 let registry = Arc::new(Registry::default());
10764 let handler = ControlHandler::new(Arc::clone(®istry));
10765 let connection = ConnectionId::new(219);
10766 registry
10767 .register_with_control_ops(
10768 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
10769 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
10770 "bad schema",
10771 format!("sha256:{}", "a".repeat(64)),
10772 128,
10773 2,
10774 )))
10775 .build(),
10776 PROTOCOL_VERSION,
10777 connection,
10778 Vec::new(),
10779 )
10780 .unwrap();
10781 let request =
10782 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
10783 let response = handler
10784 .handle_catalog_update(connection, request, Vec::new(), None, None)
10785 .unwrap();
10786 assert_eq!(response[0].header.ty, FrameType::Error);
10787 let error = parse_error(&response[0]);
10788 assert_eq!(error["code"], "invalid_manifest");
10789 assert!(error["message"]
10790 .as_str()
10791 .unwrap()
10792 .contains("rich_ui.schema"));
10793 }
10794
10795 #[test]
10796 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
10797 let registry = Arc::new(Registry::default());
10798 let supervisor = SupervisorHandle::new();
10799 supervisor.open_swap("provider", "candidate-nonce".to_string());
10800 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10801 let connection = ConnectionId::new(218);
10802 let hello = handler
10803 .handle_control(
10804 connection,
10805 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
10806 )
10807 .unwrap();
10808 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10809 assert!(registry.list_modules().unwrap().1.is_empty());
10810
10811 registry.promote_candidate("provider").unwrap().unwrap();
10812 let request = Frame::build(
10813 FrameType::Request,
10814 control_flags(),
10815 0,
10816 0,
10817 219,
10818 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10819 )
10820 .unwrap();
10821 let response = handler.handle_catalog_list(request, None).unwrap();
10822 let ClientControlResponse::CatalogList { modules, .. } =
10823 serde_json::from_slice(&response[0].body).unwrap()
10824 else {
10825 panic!("expected catalog.list");
10826 };
10827 assert_eq!(modules.len(), 1);
10828 assert!(modules[0].nonce_verified);
10829 }
10830
10831 #[test]
10832 fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
10833 for provenance_json in [
10834 r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
10835 r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
10836 ] {
10837 let registry = Arc::new(Registry::default());
10838 let handler = ControlHandler::new(registry);
10839 let hello = Frame::build(
10840 FrameType::Hello,
10841 control_flags(),
10842 0,
10843 0,
10844 220,
10845 serde_json::to_vec(&ModuleHelloBody {
10846 manifest: ModuleManifest::builder("built", "0.1.0")
10847 .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
10848 .build(),
10849 protocol_ver: PROTOCOL_VERSION,
10850 control_ops: None,
10851 launch_nonce: None,
10852 })
10853 .unwrap(),
10854 )
10855 .unwrap();
10856 assert_eq!(
10857 handler
10858 .handle_control(ConnectionId::new(220), hello)
10859 .unwrap()[0]
10860 .header
10861 .ty,
10862 FrameType::HelloAck
10863 );
10864 let request = Frame::build(
10865 FrameType::Request,
10866 control_flags(),
10867 0,
10868 0,
10869 221,
10870 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10871 )
10872 .unwrap();
10873 let response = handler.handle_catalog_list(request, None).unwrap();
10874 let ClientControlResponse::CatalogList { modules, .. } =
10875 serde_json::from_slice(&response[0].body).unwrap()
10876 else {
10877 panic!("expected catalog.list");
10878 };
10879 assert_eq!(modules.len(), 1);
10880 assert_eq!(
10881 serde_json::to_vec(&modules[0]).unwrap(),
10882 format!(
10883 r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
10884 )
10885 .as_bytes()
10886 );
10887 }
10888 }
10889
10890 #[test]
10891 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
10892 let registry = Arc::new(Registry::default());
10893 let handler = ControlHandler::new(Arc::clone(®istry));
10894 let hello = Frame::build(
10895 FrameType::Hello,
10896 control_flags(),
10897 0,
10898 0,
10899 220,
10900 serde_json::to_vec(&ModuleHelloBody {
10901 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
10902 protocol_ver: PROTOCOL_VERSION,
10903 control_ops: None,
10904 launch_nonce: Some("unverified".to_string()),
10905 })
10906 .unwrap(),
10907 )
10908 .unwrap();
10909 assert_eq!(
10910 handler
10911 .handle_control(ConnectionId::new(220), hello)
10912 .unwrap()[0]
10913 .header
10914 .ty,
10915 FrameType::HelloAck
10916 );
10917 let request = Frame::build(
10918 FrameType::Request,
10919 control_flags(),
10920 0,
10921 0,
10922 221,
10923 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10924 )
10925 .unwrap();
10926 let response = handler.handle_catalog_list(request, None).unwrap();
10927 let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
10928 assert!(wire["modules"][0].get("provenance").is_none());
10929 let ClientControlResponse::CatalogList { modules, .. } =
10930 serde_json::from_slice(&response[0].body).unwrap()
10931 else {
10932 panic!("expected catalog.list");
10933 };
10934 assert_eq!(
10935 serde_json::to_vec(&modules[0]).unwrap(),
10936 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
10937 );
10938 assert!(!modules[0].nonce_verified);
10939 assert!(modules[0].provenance.is_none());
10940 }
10941
10942 #[test]
10943 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
10944 let registry = Arc::new(Registry::default());
10945 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10946 [("vault".to_string(), true), ("squatter".to_string(), true)],
10947 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10948 );
10949 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
10950 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10951 provides: vec!["credentials-provider/v1".to_string()],
10952 requires: Vec::new(),
10953 must_never_reach: Vec::new(),
10954 });
10955 let frame = Frame::build(
10956 FrameType::Hello,
10957 control_flags(),
10958 0,
10959 0,
10960 77,
10961 serde_json::to_vec(&ModuleHelloBody {
10962 manifest: squatter,
10963 protocol_ver: PROTOCOL_VERSION,
10964 control_ops: None,
10965 launch_nonce: None,
10966 })
10967 .expect("HELLO serializes"),
10968 )
10969 .expect("HELLO frame builds");
10970 let response = handler
10971 .handle_control(ConnectionId::new(77), frame)
10972 .expect("reserved claim receives a typed refusal");
10973 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10974 assert_eq!(
10975 registry
10976 .active_registration_count()
10977 .expect("registry reads"),
10978 0,
10979 "a reserved capability refusal must not leave a catalog entry"
10980 );
10981 }
10982
10983 #[test]
10984 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10985 for settlement in ["timeout", "inconclusive", "drop"] {
10986 let breakers = RouteBindBreakers::default();
10987 let RouteBindAdmission::Admitted {
10988 guard: mut old,
10989 probe: false,
10990 } = breakers.admit("prov")
10991 else {
10992 panic!("ordinary relay admitted")
10993 };
10994 let RouteBindAdmission::Admitted {
10995 guard: mut opener, ..
10996 } = breakers.admit("prov")
10997 else {
10998 panic!("second relay admitted")
10999 };
11000 assert!(
11001 !opener
11002 .record_timeout(1, Duration::ZERO)
11003 .unwrap()
11004 .reopened_after_probe
11005 );
11006 let RouteBindAdmission::Admitted {
11007 guard: mut probe,
11008 probe: true,
11009 } = breakers.admit("prov")
11010 else {
11011 panic!("one cooldown probe admitted")
11012 };
11013 match settlement {
11014 "timeout" => assert!(
11015 !old.record_timeout(1, Duration::ZERO)
11016 .unwrap()
11017 .reopened_after_probe
11018 ),
11019 "inconclusive" => old.record_inconclusive(),
11020 "drop" => drop(old),
11021 _ => unreachable!(),
11022 }
11023 assert!(
11024 matches!(
11025 breakers.admit("prov"),
11026 RouteBindAdmission::Refused {
11027 probe_in_flight: true,
11028 ..
11029 }
11030 ),
11031 "{settlement} of a pre-open relay cannot release the real probe"
11032 );
11033 assert!(
11034 probe
11035 .record_timeout(1, Duration::ZERO)
11036 .unwrap()
11037 .reopened_after_probe
11038 );
11039 assert!(matches!(
11040 breakers.admit("prov"),
11041 RouteBindAdmission::Admitted { probe: true, .. }
11042 ));
11043 }
11044 let breakers = RouteBindBreakers::default();
11045 let admit = || match breakers.admit("prov") {
11046 RouteBindAdmission::Admitted { guard, .. } => guard,
11047 _ => panic!("relay admitted"),
11048 };
11049 admit().record_timeout(1, Duration::ZERO);
11050 let mut old_probe = admit();
11051 breakers.reset_for_new_module_connection("prov");
11052 admit().record_timeout(1, Duration::ZERO);
11053 let _new_probe = admit();
11054 old_probe.record_inconclusive();
11055 assert!(matches!(
11056 breakers.admit("prov"),
11057 RouteBindAdmission::Refused {
11058 probe_in_flight: true,
11059 ..
11060 }
11061 ));
11062 }
11063
11064 #[tokio::test]
11065 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
11066 for candidate in [false, true] {
11067 let registry = Arc::new(Registry::default());
11068 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
11069 [("vault".to_string(), true), ("squatter".to_string(), true)],
11070 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
11071 );
11072 let conn = ConnectionId::new(77);
11073 let (ctx, mut rx) = route_ctx(conn);
11074 let initial = capability_manifest("squatter", &[], &[]);
11075 if candidate {
11076 registry
11077 .register_candidate_with_control_ops(
11078 initial.clone(),
11079 PROTOCOL_VERSION,
11080 conn,
11081 module_baseline_control_ops(),
11082 )
11083 .unwrap();
11084 handler
11085 .forwarding
11086 .register_candidate_module_connection(
11087 conn,
11088 "squatter".to_string(),
11089 PROTOCOL_VERSION,
11090 manifest_concurrency(&initial),
11091 ctx.egress.clone(),
11092 )
11093 .unwrap();
11094 } else {
11095 hello_via_sink(
11096 &handler,
11097 &ctx,
11098 &mut rx,
11099 hello_frame_with_manifest(initial.clone(), 1),
11100 )
11101 .await;
11102 }
11103 let response = handler
11104 .handle_control_frame(
11105 &ctx,
11106 catalog_update_with_capabilities_frame(
11107 2,
11108 capability_manifest("squatter", &["credentials-provider/v1"], &[])
11109 .capabilities
11110 .unwrap(),
11111 ),
11112 )
11113 .await
11114 .unwrap();
11115 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
11116 assert_eq!(
11117 registry
11118 .get_module_by_connection(conn)
11119 .unwrap()
11120 .unwrap()
11121 .manifest,
11122 initial
11123 );
11124 }
11125 }
11126
11127 #[test]
11128 fn server_describe_surfaces_required_capability_verdict_fields() {
11129 let registry = Arc::new(Registry::default());
11130 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
11131 [
11132 ("consumer".to_string(), true),
11133 ("provider".to_string(), false),
11134 ],
11135 BTreeMap::new(),
11136 );
11137 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
11138 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
11139 provides: Vec::new(),
11140 requires: vec![subc_protocol::manifest::CapabilityRequirement {
11141 capability: "credentials-provider/v1".to_string(),
11142 need: subc_protocol::manifest::CapabilityNeed::Required,
11143 }],
11144 must_never_reach: Vec::new(),
11145 });
11146 let hello = Frame::build(
11147 FrameType::Hello,
11148 control_flags(),
11149 0,
11150 0,
11151 78,
11152 serde_json::to_vec(&ModuleHelloBody {
11153 manifest: consumer,
11154 protocol_ver: PROTOCOL_VERSION,
11155 control_ops: None,
11156 launch_nonce: None,
11157 })
11158 .expect("HELLO serializes"),
11159 )
11160 .expect("HELLO frame builds");
11161 handler
11162 .handle_control(ConnectionId::new(78), hello)
11163 .expect("consumer registers");
11164 let describe = Frame::build(
11165 FrameType::Request,
11166 control_flags(),
11167 0,
11168 0,
11169 79,
11170 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
11171 .expect("request serializes"),
11172 )
11173 .expect("describe frame builds");
11174 let response = handler
11175 .handle_server_describe(describe)
11176 .expect("server.describe succeeds");
11177 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
11178 let requirement = &rendered["capability_requirements"][0];
11179 assert_eq!(requirement["consumer"], "consumer");
11180 assert_eq!(requirement["verdict"], "never_provided");
11181 assert_eq!(requirement["episode_seq"], 1);
11182 assert_eq!(requirement["config_satisfiable"], false);
11183 assert_eq!(requirement["runtime_available"], false);
11184 assert!(requirement["detail"]
11185 .as_str()
11186 .expect("detail string")
11187 .contains("credentials-provider/v1"));
11188 }
11189
11190 fn deregister_required_provider(
11196 daemon_shutdown_begun: bool,
11197 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
11198 const CAPABILITY: &str = "project-identity/v1";
11199 let registry = Arc::new(Registry::default());
11200 let daemon_shutdown = DaemonShutdownFlag::default();
11201 let handler = ControlHandler::new(Arc::clone(®istry))
11202 .with_capability_config(
11203 [
11204 ("consumer".to_string(), true),
11205 ("provider".to_string(), true),
11206 ],
11207 BTreeMap::new(),
11208 )
11209 .with_daemon_shutdown_flag(daemon_shutdown.clone());
11210 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
11211 consumer.capabilities = Some(CapabilityDeclarations {
11212 provides: Vec::new(),
11213 requires: vec![subc_protocol::manifest::CapabilityRequirement {
11214 capability: CAPABILITY.to_string(),
11215 need: subc_protocol::manifest::CapabilityNeed::Required,
11216 }],
11217 must_never_reach: Vec::new(),
11218 });
11219 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
11220 let provider_conn = ConnectionId::new(182);
11221 for (conn, manifest, corr) in [
11222 (ConnectionId::new(181), consumer, 181),
11223 (provider_conn, provider, 182),
11224 ] {
11225 let hello = handler
11226 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
11227 .expect("module registers");
11228 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11229 }
11230 assert_eq!(
11231 handler.capability_evaluator.verdict("consumer", CAPABILITY),
11232 Some(CapabilityVerdict::Provided),
11233 "the registered provider must satisfy the requirement first"
11234 );
11235 if daemon_shutdown_begun {
11236 daemon_shutdown.set_for_test();
11237 }
11238
11239 let capture = EventCapture::default();
11240 let _subscriber =
11241 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11242 handler
11243 .cleanup_connection(provider_conn)
11244 .expect("provider deregisters");
11245 let events = capture
11246 .events()
11247 .into_iter()
11248 .filter(|event| {
11249 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
11250 })
11251 .collect();
11252 (
11253 events,
11254 handler.capability_evaluator.verdict("consumer", CAPABILITY),
11255 )
11256 }
11257
11258 #[test]
11259 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
11260 let (events, verdict) = deregister_required_provider(true);
11261 assert_eq!(
11262 verdict,
11263 Some(CapabilityVerdict::NeverProvided),
11264 "shutdown must not change the verdict route.open refusals read"
11265 );
11266 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11267 let event = &events[0];
11268 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
11269 assert_eq!(
11270 event.fields.get("cause").map(String::as_str),
11271 Some("\"daemon_shutdown\""),
11272 "{event:?}"
11273 );
11274 assert_eq!(
11275 event.fields.get("verdict").map(String::as_str),
11276 Some("\"never_provided\""),
11277 "{event:?}"
11278 );
11279 }
11280
11281 #[test]
11282 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
11283 let (events, verdict) = deregister_required_provider(false);
11284 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
11285 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11286 let event = &events[0];
11287 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
11288 assert_eq!(event.fields.get("cause"), None, "{event:?}");
11289 assert_eq!(
11290 event.fields.get("verdict").map(String::as_str),
11291 Some("\"never_provided\""),
11292 "{event:?}"
11293 );
11294 }
11295
11296 #[test]
11297 fn catalog_list_omits_capabilities_for_legacy_manifest() {
11298 let registry = Arc::new(Registry::default());
11299 let handler = ControlHandler::new(Arc::clone(®istry));
11300 let hello = handler
11301 .handle_control(
11302 ConnectionId::new(101),
11303 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
11304 )
11305 .expect("legacy HELLO registers");
11306 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11307
11308 let request = Frame::build(
11309 FrameType::Request,
11310 control_flags(),
11311 0,
11312 0,
11313 102,
11314 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
11315 .expect("catalog request serializes"),
11316 )
11317 .expect("catalog request frame builds");
11318 let response = handler
11319 .handle_catalog_list(request, None)
11320 .expect("catalog list succeeds");
11321 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
11322 assert!(
11323 body["modules"][0].get("capabilities").is_none(),
11324 "legacy manifest must retain an absent capabilities field on catalog.list"
11325 );
11326 }
11327
11328 #[test]
11329 fn hello_ack_omits_storage_when_no_storage_config() {
11330 let registry = Arc::new(Registry::default());
11331 let handler = ControlHandler::new(Arc::clone(®istry));
11332 let responses = handler
11333 .handle_control(
11334 ConnectionId::new(1),
11335 hello_frame("aft", PROTOCOL_VERSION, 7),
11336 )
11337 .unwrap();
11338 let ack = parse_ack(&responses[0]);
11339 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
11340 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
11341 }
11342
11343 #[tokio::test]
11344 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
11345 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
11346 let registry = Arc::new(Registry::default());
11347 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
11348 let responses = handler
11349 .handle_control(
11350 ConnectionId::new(1),
11351 hello_frame("aft", PROTOCOL_VERSION, 7),
11352 )
11353 .unwrap();
11354 let ack = parse_ack(&responses[0]);
11355 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
11356
11357 let described = handler
11358 .handle_control_frame(
11359 &route_ctx(ConnectionId::new(2)).0,
11360 Frame::build(
11361 FrameType::Request,
11362 control_flags(),
11363 0,
11364 0,
11365 9,
11366 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11367 )
11368 .unwrap(),
11369 )
11370 .await
11371 .unwrap();
11372 let ClientControlResponse::ServerDescribe { machine_id, .. } =
11373 serde_json::from_slice(&described[0].body).unwrap()
11374 else {
11375 panic!("server.describe answered with another shape");
11376 };
11377 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
11378 }
11379
11380 #[test]
11381 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
11382 let registry = Arc::new(Registry::default());
11385 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
11386 crate::daemon_config::StorageConfig::Sqlite {
11387 data_home: std::path::PathBuf::from("/data"),
11388 },
11389 ));
11390
11391 let responses = handler
11392 .handle_control(
11393 ConnectionId::new(1),
11394 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
11395 )
11396 .unwrap();
11397 let ack = parse_ack(&responses[0]);
11398 assert_eq!(
11399 ack.storage,
11400 Some(serde_json::json!({
11401 "module_id": "alfonso-routing",
11402 "storage_namespace": "default",
11403 "isolation": { "kind": "module" },
11404 "backend": {
11405 "backend": "sqlite",
11406 "path": "/data/cortexkit/alfonso-routing/store.db"
11407 }
11408 })),
11409 "the delivered descriptor is the module's own sqlite store path"
11410 );
11411 }
11412
11413 #[test]
11414 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
11415 let registry = Arc::new(Registry::default());
11416 let handler = ControlHandler::new(Arc::clone(®istry));
11417 let conn = ConnectionId::new(1);
11418 let responses = handler
11419 .handle_control(
11420 conn,
11421 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11422 )
11423 .unwrap();
11424 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11425 let registration = registry.get_module("aft").unwrap().unwrap();
11426 assert_eq!(registration.control_ops, module_baseline_control_ops());
11427
11428 let frame =
11429 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
11430 assert!(handler
11431 .guard_module_control_op(&frame, "aft", "route.bind")
11432 .unwrap()
11433 .is_none());
11434 let error = handler
11435 .guard_module_control_op(&frame, "aft", "test.synthetic")
11436 .unwrap()
11437 .expect("synthetic ungranted op should be rejected");
11438 assert_eq!(error.header.ty, FrameType::Error);
11439 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
11440 }
11441
11442 #[test]
11443 fn hello_control_ops_some_adds_optional_grants() {
11444 let registry = Arc::new(Registry::default());
11445 let handler = ControlHandler::new(Arc::clone(®istry));
11446 handler
11447 .handle_control(
11448 ConnectionId::new(1),
11449 hello_frame_with_control_ops(
11450 "aft",
11451 PROTOCOL_VERSION,
11452 7,
11453 Some(vec![
11454 "future.synthetic".to_string(),
11455 "route.bind".to_string(),
11456 ]),
11457 ),
11458 )
11459 .unwrap();
11460 let registration = registry.get_module("aft").unwrap().unwrap();
11461 assert_eq!(
11462 registration.control_ops,
11463 vec![
11464 "route.bind".to_string(),
11465 "route.status".to_string(),
11466 "future.synthetic".to_string(),
11467 ]
11468 );
11469 let frame =
11470 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
11471 assert!(handler
11472 .guard_module_control_op(&frame, "aft", "future.synthetic")
11473 .unwrap()
11474 .is_none());
11475 }
11476
11477 #[tokio::test]
11478 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
11479 let registry = Arc::new(Registry::default());
11480 let forwarding = Arc::new(ForwardingTable::default());
11481 let handler =
11482 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11483 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
11484 hello_via_sink(
11485 &handler,
11486 &module_ctx,
11487 &mut module_rx,
11488 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11489 )
11490 .await;
11491
11492 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
11493 let responses = handler
11494 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
11495 .await
11496 .unwrap();
11497 assert_eq!(responses.len(), 1);
11498 assert_eq!(responses[0].header.ty, FrameType::Error);
11499 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
11500 assert!(module_rx.try_recv().is_err());
11501 }
11502
11503 #[tokio::test]
11504 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
11505 let registry = Arc::new(Registry::default());
11506 let forwarding = Arc::new(ForwardingTable::default());
11507 let handler =
11508 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11509 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
11510 hello_via_sink(
11511 &handler,
11512 &module_ctx,
11513 &mut module_rx,
11514 hello_frame_with_control_ops(
11515 "aft",
11516 PROTOCOL_VERSION,
11517 7,
11518 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11519 ),
11520 )
11521 .await;
11522
11523 let project_root = unique_project_root("demux");
11524 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
11525 let route_handler = handler.clone();
11526 let route_task = tokio::spawn(async move {
11527 route_handler
11528 .handle_control_frame(
11529 &route_client_ctx,
11530 route_open_frame(100, "aft", project_root),
11531 )
11532 .await
11533 .unwrap()
11534 });
11535 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11536 .await
11537 .unwrap()
11538 .unwrap();
11539 assert!(matches!(
11540 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
11541 ModuleControlRequest::RouteBind { .. }
11542 ));
11543
11544 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
11545 let health_handler = handler.clone();
11546 let health_task = tokio::spawn(async move {
11547 health_handler
11548 .handle_control_frame(
11549 &health_client_ctx,
11550 supervisor_health_probe_frame(101, "aft"),
11551 )
11552 .await
11553 .unwrap()
11554 });
11555 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11556 .await
11557 .unwrap()
11558 .unwrap();
11559 assert_eq!(
11560 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11561 ModuleControlRequest::HealthCheck {}
11562 );
11563
11564 handler
11565 .handle_control_frame(
11566 &module_ctx,
11567 health_response(health_frame.header.corr, HealthStatus::Degraded),
11568 )
11569 .await
11570 .unwrap();
11571 let health_response = health_task.await.unwrap();
11572 assert_eq!(health_response.len(), 1);
11573 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11574 ClientControlResponse::SupervisorHealthProbe {
11575 module_id,
11576 status,
11577 detail,
11578 metrics,
11579 } => {
11580 assert_eq!(module_id, "aft");
11581 assert_eq!(status, HealthStatus::Degraded);
11582 assert_eq!(detail.as_deref(), Some("warming"));
11583 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
11584 }
11585 other => panic!("unexpected health response: {other:?}"),
11586 }
11587
11588 handler
11589 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11590 .await
11591 .unwrap();
11592 let route_response = route_task.await.unwrap();
11593 assert!(route_response.is_empty());
11594 let published = route_client_rx.recv().await.unwrap();
11595 assert!(matches!(
11596 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11597 ClientControlResponse::RouteOpen { .. }
11598 ));
11599 }
11600
11601 async fn relay_route_open(
11606 handler: &ControlHandler,
11607 client_connection: ConnectionId,
11608 client_egress: &FrameSink,
11609 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11610 corr: u64,
11611 module_id: &str,
11612 project_root_label: &str,
11613 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
11614 let ctx = RouteCtx {
11615 connection_id: client_connection,
11616 egress: client_egress.clone(),
11617 };
11618 let handler = handler.clone();
11619 let project_root = unique_project_root(project_root_label);
11620 let module_id = module_id.to_string();
11621 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
11622 let task = tokio::spawn(async move {
11623 let _guard = tracing::dispatcher::set_default(&dispatch);
11624 handler
11625 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
11626 .await
11627 .unwrap()
11628 });
11629 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
11630 .await
11631 .expect("module receives the relayed route.bind")
11632 .expect("module egress is open");
11633 (task, bind.frame)
11634 }
11635
11636 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
11637 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
11638 ModuleControlRequest::RouteBind {
11639 route_channel,
11640 epoch,
11641 ..
11642 } => (route_channel, epoch),
11643 other => panic!("expected a route.bind request, got {other:?}"),
11644 }
11645 }
11646
11647 fn published_route(frame: &Frame) -> (u16, u32) {
11648 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
11649 ClientControlResponse::RouteOpen {
11650 route_channel,
11651 route_epoch,
11652 ..
11653 } => (route_channel, route_epoch),
11654 other => panic!("expected a route.open response, got {other:?}"),
11655 }
11656 }
11657
11658 #[tokio::test]
11678 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
11679 let registry = Arc::new(Registry::default());
11680 let forwarding = Arc::new(ForwardingTable::default());
11681 let handler =
11682 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11683
11684 let module_connection = ConnectionId::new(30);
11685 let (module_ctx, mut module_rx) = route_ctx(module_connection);
11686 hello_via_sink(
11687 &handler,
11688 &module_ctx,
11689 &mut module_rx,
11690 hello_frame("aft", PROTOCOL_VERSION, 7),
11691 )
11692 .await;
11693
11694 let dying_client = ConnectionId::new(31);
11695 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
11696
11697 let (first_task, first_bind) = relay_route_open(
11700 &handler,
11701 dying_client,
11702 &dying_ctx.egress,
11703 &mut module_rx,
11704 100,
11705 "aft",
11706 "closing-first",
11707 )
11708 .await;
11709 handler
11710 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
11711 .await
11712 .unwrap();
11713 assert!(first_task.await.unwrap().is_empty());
11714 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
11715
11716 let (second_task, second_bind) = relay_route_open(
11718 &handler,
11719 dying_client,
11720 &dying_ctx.egress,
11721 &mut module_rx,
11722 101,
11723 "aft",
11724 "closing-second",
11725 )
11726 .await;
11727 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
11728
11729 assert!(forwarding
11732 .escalate_client_delivery_failure(
11733 dying_client,
11734 first_channel,
11735 first_epoch,
11736 CloseReason::new(
11737 "module_to_client_delivery_failed",
11738 "client egress refused a module frame",
11739 ),
11740 crate::forwarding::UndeliveredFrame {
11741 module_id: None,
11742 sink: &dying_ctx.egress,
11743 },
11744 )
11745 .unwrap());
11746 assert!(!dying_ctx.egress.is_closed());
11747
11748 let ack = handler
11750 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
11751 .await;
11752 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11753 if module_loop_error.is_some() {
11754 handler.cleanup_connection(module_connection).unwrap();
11758 }
11759 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11763 .await
11764 .ok()
11765 .flatten();
11766
11767 assert!(
11769 registry
11770 .get_module_by_connection(module_connection)
11771 .unwrap()
11772 .is_some(),
11773 "one client's closing connection ended the shared module connection: \
11774 {module_loop_error:?}"
11775 );
11776 let cotenant = ConnectionId::new(32);
11778 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11779 let (cotenant_task, cotenant_bind) = relay_route_open(
11780 &handler,
11781 cotenant,
11782 &cotenant_ctx.egress,
11783 &mut module_rx,
11784 102,
11785 "aft",
11786 "closing-cotenant",
11787 )
11788 .await;
11789 handler
11790 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
11791 .await
11792 .unwrap();
11793 assert!(cotenant_task.await.unwrap().is_empty());
11794 let (cotenant_channel, cotenant_epoch) =
11795 published_route(&cotenant_rx.recv().await.unwrap());
11796 assert!(matches!(
11797 forwarding
11798 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11799 .unwrap(),
11800 DataRoute::Client(DataRouteState::Bound(_))
11801 ));
11802
11803 let goodbye = post_ack_module_frame
11806 .expect("module receives a GOODBYE for the abandoned route channel");
11807 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11808 assert_eq!(goodbye.header.channel, abandoned_channel);
11809 assert_eq!(goodbye.header.epoch, abandoned_epoch);
11810
11811 assert!(dying_rx.try_recv().is_err());
11815 let second_response = second_task.await.unwrap();
11816 assert_eq!(second_response.len(), 1);
11817 assert_eq!(
11818 parse_error(&second_response[0])["code"],
11819 "target_unavailable"
11820 );
11821 }
11822
11823 async fn catalog_snapshot(
11826 handler: &ControlHandler,
11827 client_ctx: &RouteCtx,
11828 corr: u64,
11829 ) -> (u64, Vec<(String, bool)>) {
11830 let body =
11831 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
11832 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
11833 let reply = handler
11834 .handle_control_frame(client_ctx, request)
11835 .await
11836 .unwrap()
11837 .pop()
11838 .unwrap();
11839 let ClientControlResponse::CatalogList {
11840 generation,
11841 modules,
11842 ..
11843 } = serde_json::from_slice(&reply.body).unwrap()
11844 else {
11845 panic!("not a catalog.list reply");
11846 };
11847 (
11848 generation,
11849 modules
11850 .into_iter()
11851 .map(|entry| (entry.module_id, entry.ready))
11852 .collect(),
11853 )
11854 }
11855
11856 fn hello_held_before_forwarding() -> (
11862 ControlHandler,
11863 Arc<std::sync::Barrier>,
11864 std::thread::JoinHandle<Vec<Frame>>,
11865 RouteCtx,
11866 mpsc::Receiver<crate::router::OutboundFrame>,
11867 ) {
11868 let hold = Arc::new(std::sync::Barrier::new(2));
11869 let handler = ControlHandler::with_forwarding(
11870 Arc::new(Registry::default()),
11871 Arc::new(ForwardingTable::default()),
11872 )
11873 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
11874 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11875 let hello = {
11876 let handler = handler.clone();
11877 let module_ctx = module_ctx.clone();
11878 std::thread::spawn(move || {
11879 handler
11880 .handle_hello(
11881 module_ctx.connection_id,
11882 Some(module_ctx.egress.clone()),
11883 hello_frame("aft", PROTOCOL_VERSION, 7),
11884 )
11885 .unwrap()
11886 })
11887 };
11888 (handler, hold, hello, module_ctx, module_rx)
11889 }
11890
11891 #[tokio::test]
11900 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
11901 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
11902 let client = ConnectionId::new(41);
11903 let (client_ctx, mut client_rx) = route_ctx(client);
11904
11905 hold.wait();
11906 assert!(handler.registry.get_module("aft").unwrap().is_some());
11908 assert!(!handler
11909 .forwarding
11910 .has_live_module_connection("aft")
11911 .unwrap());
11912 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
11913 assert!(
11914 listed.is_empty(),
11915 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
11916 );
11917 let refused = handler
11918 .handle_control_frame(
11919 &client_ctx,
11920 route_open_frame(301, "aft", unique_project_root("listed-window")),
11921 )
11922 .await
11923 .unwrap();
11924 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
11925 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
11926 let duplicate = handler
11927 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
11928 .await
11929 .unwrap();
11930 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11931
11932 hold.wait();
11933 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11934 let ack = module_rx.try_recv().unwrap().frame;
11935 assert_eq!(ack.header.ty, FrameType::HelloAck);
11936 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11937 assert_eq!(listed, vec![("aft".to_string(), true)]);
11938
11939 let (open_task, bind) = relay_route_open(
11940 &handler,
11941 client,
11942 &client_ctx.egress,
11943 &mut module_rx,
11944 303,
11945 "aft",
11946 "listed-after",
11947 )
11948 .await;
11949 assert!(handler
11950 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11951 .await
11952 .unwrap()
11953 .is_empty());
11954 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
11955 published_route(&client_rx.try_recv().unwrap().frame);
11956 }
11957
11958 #[tokio::test]
11965 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11966 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11967 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11968 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11969
11970 hold.wait();
11971 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11972 assert!(listed.is_empty());
11973 assert_eq!(
11974 during, before,
11975 "the generation moved while the module was not listed"
11976 );
11977
11978 hold.wait();
11979 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11980 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11981 assert_eq!(listed, vec![("aft".to_string(), true)]);
11982 assert_eq!(
11983 after,
11984 before + 1,
11985 "the module appeared without exactly one generation bump"
11986 );
11987 }
11988
11989 #[tokio::test]
11994 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11995 let registry = Arc::new(Registry::default());
11996 let handler = ControlHandler::with_forwarding(
11997 Arc::clone(®istry),
11998 Arc::new(ForwardingTable::default()),
11999 );
12000 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12001 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
12002 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
12003 drop(module_rx);
12004
12005 let replies = handler
12006 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
12007 .await
12008 .unwrap();
12009 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
12010 assert!(registry.get_module("aft").unwrap().is_none());
12011 assert!(registry
12012 .get_module_by_connection(module_ctx.connection_id)
12013 .unwrap()
12014 .is_none());
12015 assert_eq!(
12016 catalog_snapshot(&handler, &client_ctx, 301).await,
12017 (before, Vec::new())
12018 );
12019 }
12020
12021 async fn route_open_held_after_bind_send() -> (
12025 ControlHandler,
12026 Arc<tokio::sync::Barrier>,
12027 RouteCtx,
12028 mpsc::Receiver<crate::router::OutboundFrame>,
12029 ) {
12030 let hold = Arc::new(tokio::sync::Barrier::new(2));
12031 let handler = ControlHandler::with_forwarding(
12032 Arc::new(Registry::default()),
12033 Arc::new(ForwardingTable::default()),
12034 )
12035 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
12036 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12037 hello_via_sink(
12038 &handler,
12039 &module_ctx,
12040 &mut module_rx,
12041 hello_frame("aft", PROTOCOL_VERSION, 7),
12042 )
12043 .await;
12044 (handler, hold, module_ctx, module_rx)
12045 }
12046
12047 #[tokio::test]
12054 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
12055 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12056 let client = ConnectionId::new(41);
12057 let (client_ctx, mut client_rx) = route_ctx(client);
12058
12059 let (open_task, bind) = relay_route_open(
12060 &handler,
12061 client,
12062 &client_ctx.egress,
12063 &mut module_rx,
12064 200,
12065 "aft",
12066 "acked-before-mark",
12067 )
12068 .await;
12069 let (module_channel, module_epoch) = route_bind_channel(&bind);
12070 hold.wait().await;
12072
12073 assert!(handler
12075 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12076 .await
12077 .unwrap()
12078 .is_empty());
12079 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
12080
12081 hold.wait().await;
12082 assert!(
12083 open_task.await.unwrap().is_empty(),
12084 "route.open answers accepted"
12085 );
12086
12087 match module_rx.try_recv() {
12090 Err(mpsc::error::TryRecvError::Empty) => {}
12091 Ok(outbound) => panic!(
12092 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
12093 outbound.frame.header.ty,
12094 outbound.frame.header.channel,
12095 outbound.frame.header.epoch
12096 ),
12097 Err(err) => panic!("module egress closed: {err}"),
12098 }
12099
12100 let router = Router::with_control_handler(Arc::new(handler.clone()));
12103 let request = Frame::build(
12104 FrameType::Request,
12105 Flags::new(true, Priority::Interactive, false),
12106 client_channel,
12107 client_epoch,
12108 201,
12109 b"first request".to_vec(),
12110 )
12111 .unwrap();
12112 router
12113 .route_for_connection(&client_ctx, request)
12114 .await
12115 .unwrap();
12116 let delivered = module_rx.try_recv().unwrap().frame;
12117 assert_eq!(delivered.header.ty, FrameType::Request);
12118 assert_eq!(
12119 (delivered.header.channel, delivered.header.epoch),
12120 (module_channel, module_epoch)
12121 );
12122 assert_eq!(delivered.body, b"first request");
12123 }
12124
12125 #[tokio::test]
12131 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
12132 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12133 let client = ConnectionId::new(42);
12134 let (client_ctx, _client_rx) = route_ctx(client);
12135
12136 let (open_task, bind) = relay_route_open(
12137 &handler,
12138 client,
12139 &client_ctx.egress,
12140 &mut module_rx,
12141 300,
12142 "aft",
12143 "released-before-mark",
12144 )
12145 .await;
12146 let (module_channel, module_epoch) = route_bind_channel(&bind);
12147 hold.wait().await;
12148
12149 handler.cleanup_connection(client).unwrap();
12150 assert!(
12151 module_rx.try_recv().is_err(),
12152 "client teardown must not GOODBYE a bind route.open is still queueing"
12153 );
12154
12155 hold.wait().await;
12156 let response = open_task.await.unwrap();
12157 assert_eq!(response.len(), 1);
12158 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12159
12160 let goodbye = module_rx
12161 .try_recv()
12162 .expect("module is told to drop the binding the daemon abandoned")
12163 .frame;
12164 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12165 assert_eq!(
12166 (goodbye.header.channel, goodbye.header.epoch),
12167 (module_channel, module_epoch)
12168 );
12169 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
12170 }
12171
12172 #[tokio::test]
12177 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
12178 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12179 let client = ConnectionId::new(43);
12180 let (client_ctx, client_rx) = route_ctx(client);
12181
12182 let (open_task, bind) = relay_route_open(
12183 &handler,
12184 client,
12185 &client_ctx.egress,
12186 &mut module_rx,
12187 400,
12188 "aft",
12189 "gone-client-before-mark",
12190 )
12191 .await;
12192 let (module_channel, module_epoch) = route_bind_channel(&bind);
12193 hold.wait().await;
12194
12195 drop(client_rx);
12196 assert!(client_ctx.egress.is_closed());
12197 handler
12198 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12199 .await
12200 .unwrap();
12201
12202 hold.wait().await;
12203 let response = open_task.await.unwrap();
12204 assert_eq!(response.len(), 1);
12205 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12206
12207 let goodbye = module_rx
12208 .try_recv()
12209 .expect("module is told to drop the binding it acked")
12210 .frame;
12211 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12212 assert_eq!(
12213 (goodbye.header.channel, goodbye.header.epoch),
12214 (module_channel, module_epoch)
12215 );
12216 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
12217 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
12218 }
12219
12220 #[test]
12227 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
12228 let handler = ControlHandler::default();
12229 let module_connection = ConnectionId::new(30);
12230 let client_connection = ConnectionId::new(31);
12231
12232 handler
12233 .refuse_to_end_module_connection_for_a_client(
12234 module_connection,
12235 77,
12236 ForwardingError::ConnectionClosing {
12237 connection_id: client_connection,
12238 },
12239 )
12240 .expect("a closing client must never end the module connection");
12241
12242 assert!(matches!(
12243 handler.refuse_to_end_module_connection_for_a_client(
12244 module_connection,
12245 78,
12246 ForwardingError::ConnectionClosing {
12247 connection_id: module_connection,
12248 },
12249 ),
12250 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
12251 connection_id
12252 })) if connection_id == module_connection
12253 ));
12254 assert!(matches!(
12255 handler.refuse_to_end_module_connection_for_a_client(
12256 module_connection,
12257 79,
12258 ForwardingError::Poisoned,
12259 ),
12260 Err(RouterError::Forwarding(ForwardingError::Poisoned))
12261 ));
12262 assert!(matches!(
12263 handler.refuse_to_end_module_connection_for_a_client(
12264 module_connection,
12265 80,
12266 ForwardingError::StaleModuleEndpoint,
12267 ),
12268 Err(RouterError::Forwarding(
12269 ForwardingError::StaleModuleEndpoint
12270 ))
12271 ));
12272 }
12273
12274 #[tokio::test]
12287 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
12288 let registry = Arc::new(Registry::default());
12289 let forwarding = Arc::new(ForwardingTable::default());
12290 let supervisor = SupervisorHandle::new();
12291 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12292 let handler =
12293 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12294 .with_supervisor(supervisor);
12295
12296 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12297 hello_via_sink(
12298 &handler,
12299 &target_ctx,
12300 &mut target_rx,
12301 hello_frame("target", PROTOCOL_VERSION, 1),
12302 )
12303 .await;
12304
12305 let wrong_nonce = handler
12309 .handle_control_frame(
12310 &route_ctx(ConnectionId::new(91)).0,
12311 route_open_frame_with_admission_facts(
12312 20,
12313 "target",
12314 unique_project_root("admission-facts"),
12315 Some(subc_control::ConsumerIdentity {
12316 module_id: "fed".to_string(),
12317 launch_nonce: "not-the-real-nonce".to_string(),
12318 }),
12319 None,
12320 ),
12321 )
12322 .await
12323 .unwrap();
12324 assert_eq!(
12325 parse_error(&wrong_nonce[0])["code"],
12326 "bad_consumer_identity",
12327 "a mismatched launch nonce must be refused, not stamped Reserved"
12328 );
12329
12330 let never_spawned = handler
12334 .handle_control_frame(
12335 &route_ctx(ConnectionId::new(92)).0,
12336 route_open_frame_with_admission_facts(
12337 21,
12338 "target",
12339 unique_project_root("admission-facts"),
12340 Some(subc_control::ConsumerIdentity {
12341 module_id: "never-spawned".to_string(),
12342 launch_nonce: "any-nonce".to_string(),
12343 }),
12344 None,
12345 ),
12346 )
12347 .await
12348 .unwrap();
12349 assert_eq!(
12350 parse_error(&never_spawned[0])["code"],
12351 "bad_consumer_identity",
12352 "an unspawned module_id must be refused rather than accepted for lack of a record"
12353 );
12354 }
12355
12356 #[tokio::test]
12379 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
12380 let registry = Arc::new(Registry::default());
12381 let forwarding = Arc::new(ForwardingTable::default());
12382 let supervisor = SupervisorHandle::new();
12383 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12384 let handler =
12385 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12386 .with_supervisor(supervisor);
12387
12388 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
12389 hello_via_sink(
12390 &handler,
12391 &target_ctx,
12392 &mut target_rx,
12393 hello_frame("target", PROTOCOL_VERSION, 1),
12394 )
12395 .await;
12396
12397 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
12398 let route_handler = handler.clone();
12399 let route_task = tokio::spawn(async move {
12400 route_handler
12401 .handle_control_frame(
12402 &client_ctx,
12403 route_open_frame_with_admission_facts(
12404 30,
12405 "target",
12406 unique_project_root("admission-facts"),
12407 Some(subc_control::ConsumerIdentity {
12408 module_id: "fed".to_string(),
12409 launch_nonce: "fed-nonce".to_string(),
12410 }),
12411 None,
12412 ),
12413 )
12414 .await
12415 .unwrap()
12416 });
12417
12418 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12425 .await
12426 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
12427 .expect("module control channel closed before route.bind");
12428 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12429 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12430 panic!("expected route.bind")
12431 };
12432 assert_eq!(
12433 principal,
12434 Some(Principal::Reserved {
12435 module_id: "fed".to_string()
12436 }),
12437 "a correctly attested consumer must be stamped Reserved for its own id"
12438 );
12439
12440 handler
12441 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12442 .await
12443 .unwrap();
12444 assert!(route_task.await.unwrap().is_empty());
12445 assert!(
12446 matches!(
12447 serde_json::from_slice::<ClientControlResponse>(
12448 &client_rx.recv().await.unwrap().body
12449 )
12450 .unwrap(),
12451 ClientControlResponse::RouteOpen { .. }
12452 ),
12453 "the route must actually open, not merely avoid an error"
12454 );
12455 }
12456
12457 #[tokio::test(start_paused = true)]
12458 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
12459 let registry = Arc::new(Registry::default());
12460 let forwarding = Arc::new(ForwardingTable::default());
12461 let supervisor = SupervisorHandle::new();
12462 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12463 let handler =
12464 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12465 .with_supervisor(supervisor);
12466
12467 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
12468 hello_via_sink(
12469 &handler,
12470 &target_ctx,
12471 &mut target_rx,
12472 hello_frame("target", PROTOCOL_VERSION, 1),
12473 )
12474 .await;
12475
12476 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
12477 let direct_handler = handler.clone();
12478 let direct_open = tokio::spawn(async move {
12479 direct_handler
12480 .handle_control_frame(
12481 &direct_ctx,
12482 route_open_frame(2, "target", unique_project_root("route-census-direct")),
12483 )
12484 .await
12485 .unwrap()
12486 });
12487 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12488 .await
12489 .expect("no direct route.bind within 5s")
12490 .expect("target control channel closed before direct route.bind");
12491 handler
12492 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
12493 .await
12494 .unwrap();
12495 assert!(direct_open.await.unwrap().is_empty());
12496 let _ = direct_rx.recv().await.unwrap();
12497
12498 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
12499 let reserved_handler = handler.clone();
12500 let reserved_open = tokio::spawn(async move {
12501 reserved_handler
12502 .handle_control_frame(
12503 &reserved_ctx,
12504 route_open_frame_with_admission_facts(
12505 3,
12506 "target",
12507 unique_project_root("admission-facts"),
12508 Some(ConsumerIdentity {
12509 module_id: "fed".to_string(),
12510 launch_nonce: "fed-nonce".to_string(),
12511 }),
12512 None,
12513 ),
12514 )
12515 .await
12516 .unwrap()
12517 });
12518 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12519 .await
12520 .expect("no reserved route.bind within 5s")
12521 .expect("target control channel closed before reserved route.bind");
12522 handler
12523 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
12524 .await
12525 .unwrap();
12526 assert!(reserved_open.await.unwrap().is_empty());
12527 let _ = reserved_rx.recv().await.unwrap();
12528
12529 forwarding
12530 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
12531 .unwrap();
12532 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
12533 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
12534 module_id: Some("target".to_string()),
12535 })
12536 .unwrap();
12537 let census_frame =
12538 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
12539 let response = handler
12540 .handle_control_frame(&census_ctx, census_frame)
12541 .await
12542 .unwrap()
12543 .pop()
12544 .unwrap();
12545 let actual: Value = serde_json::from_slice(&response.body).unwrap();
12546 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
12547 assert!(matches!(
12548 decoded,
12549 ClientControlResponse::SupervisorRoutes { .. }
12550 ));
12551 let routes = actual["modules"][0]["routes"].as_array().unwrap();
12552 assert_eq!(routes.len(), 2);
12553 assert!(routes.iter().all(|route| route["draining"] == true));
12554 assert!(
12557 routes.iter().all(|route| route["drain_reason"] == "reload"),
12558 "draining routes must name the drain's reason: {routes:?}"
12559 );
12560 assert!(routes.iter().any(|route| {
12561 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
12562 }));
12563 assert!(routes.iter().any(|route| {
12564 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
12565 }));
12566
12567 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12568 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
12569 if std::env::var_os("UPDATE_GOLDEN").is_some() {
12570 std::fs::write(
12571 &golden_path,
12572 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
12573 )
12574 .unwrap();
12575 }
12576 let expected: Value =
12577 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
12578 assert_eq!(actual, expected);
12579 }
12580
12581 async fn query_live_roots(
12582 handler: &ControlHandler,
12583 module_ctx: &RouteCtx,
12584 ) -> ModuleControlResponseToModule {
12585 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
12586 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
12587 let response = handler
12588 .handle_control_frame(module_ctx, frame)
12589 .await
12590 .unwrap()
12591 .pop()
12592 .unwrap();
12593 serde_json::from_slice(&response.body).unwrap()
12594 }
12595
12596 #[tokio::test(start_paused = true)]
12597 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
12598 let registry = Arc::new(Registry::default());
12599 let forwarding = Arc::new(ForwardingTable::default());
12600 let handler = ControlHandler::with_forwarding(registry, forwarding);
12601 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
12602 hello_via_sink(
12603 &handler,
12604 &target_ctx,
12605 &mut target_rx,
12606 hello_frame("target", PROTOCOL_VERSION, 1),
12607 )
12608 .await;
12609 let root = unique_project_root("live-roots-known");
12610 let path = ProjectRootId::from_path_allowing_missing(root.path())
12611 .unwrap()
12612 .as_path()
12613 .to_path_buf();
12614 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
12615 let open_handler = handler.clone();
12616 let opened = tokio::spawn(async move {
12617 open_handler
12618 .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
12619 .await
12620 .unwrap()
12621 });
12622 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12623 .await
12624 .unwrap()
12625 .unwrap();
12626 handler
12627 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
12628 .await
12629 .unwrap();
12630 assert!(opened.await.unwrap().is_empty());
12631 let _ = client_rx.recv().await.unwrap();
12632
12633 let root = unique_project_root("live-roots-pending");
12634 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
12635 .unwrap()
12636 .as_path()
12637 .to_path_buf();
12638 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
12639 let open_handler = handler.clone();
12640 let pending = tokio::spawn(async move {
12641 open_handler
12642 .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
12643 .await
12644 .unwrap()
12645 });
12646 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12647 .await
12648 .unwrap()
12649 .unwrap();
12650 let actual = query_live_roots(&handler, &target_ctx).await;
12651 let ModuleControlResponseToModule::LiveRoots {
12652 roots,
12653 unknown_root_bindings,
12654 total_bindings,
12655 } = actual
12656 else {
12657 panic!("expected live roots")
12658 };
12659 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
12660 assert_eq!(unknown_root_bindings, 0);
12661 assert_eq!(
12662 roots.len(),
12663 2,
12664 "root-known arm must retain each canonical root"
12665 );
12666 assert_eq!(
12667 total_bindings,
12668 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12669 );
12670 let counts = roots
12671 .iter()
12672 .map(|root| (root.project_root.clone(), root.bound, root.pending))
12673 .collect::<Vec<_>>();
12674 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
12675 expected.sort_by(|a, b| a.0.cmp(&b.0));
12676 assert_eq!(
12677 counts, expected,
12678 "roots must sort by path and count pending separately"
12679 );
12680 handler
12681 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
12682 .await
12683 .unwrap();
12684 assert!(pending.await.unwrap().is_empty());
12685 }
12686
12687 #[tokio::test(start_paused = true)]
12688 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
12689 let forwarding = Arc::new(ForwardingTable::default());
12690 let handler =
12691 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12692 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
12693 hello_via_sink(
12694 &handler,
12695 &target_ctx,
12696 &mut target_rx,
12697 hello_frame("target", PROTOCOL_VERSION, 1),
12698 )
12699 .await;
12700 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
12701 let pending = forwarding
12702 .begin_route_bind_relay_for_test(
12703 client_ctx.connection_id,
12704 client_ctx.egress.clone(),
12705 2,
12706 "target",
12707 )
12708 .unwrap();
12709 forwarding
12710 .complete_pending_relay(
12711 target_ctx.connection_id,
12712 pending.corr,
12713 RouteBindRelayOutcome::Accepted,
12714 )
12715 .unwrap();
12716 let actual = query_live_roots(&handler, &target_ctx).await;
12717 let ModuleControlResponseToModule::LiveRoots {
12718 roots,
12719 unknown_root_bindings,
12720 total_bindings,
12721 } = actual
12722 else {
12723 panic!("expected live roots")
12724 };
12725 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
12726 assert_eq!(
12727 unknown_root_bindings, 1,
12728 "unknown-root arm must not read as no bindings"
12729 );
12730 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
12731 assert_eq!(
12732 total_bindings,
12733 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12734 );
12735 }
12736
12737 #[tokio::test(start_paused = true)]
12745 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
12746 let forwarding = Arc::new(ForwardingTable::default());
12747 let handler =
12748 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12749 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
12750 let replies = handler
12751 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
12752 .await
12753 .unwrap();
12754 let queued_by_hello = module_rx.len();
12755
12756 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
12757 let open_handler = handler.clone();
12758 let open = tokio::spawn(async move {
12759 open_handler
12760 .handle_control_frame(
12761 &client_ctx,
12762 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
12763 )
12764 .await
12765 .unwrap()
12766 });
12767 let mut spins = 0;
12769 while module_rx.len() == queued_by_hello {
12770 spins += 1;
12771 assert!(spins < 10_000, "route.open never queued a route.bind");
12772 tokio::task::yield_now().await;
12773 }
12774
12775 for reply in replies {
12777 module_ctx.egress.send(reply).await.unwrap();
12778 }
12779
12780 let first = module_rx.recv().await.unwrap().frame;
12781 assert_eq!(
12782 first.header.ty,
12783 FrameType::HelloAck,
12784 "the first frame a registering module reads must be its HELLO_ACK"
12785 );
12786 assert_eq!(first.header.corr, 7);
12787 let second = module_rx.recv().await.unwrap().frame;
12788 assert_eq!(second.header.ty, FrameType::Request);
12789 assert!(
12790 matches!(
12791 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
12792 ModuleControlRequest::RouteBind { .. }
12793 ),
12794 "the route.bind follows the ack"
12795 );
12796 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
12797
12798 handler
12799 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
12800 .await
12801 .unwrap();
12802 assert!(open.await.unwrap().is_empty());
12803 let _ = client_rx.recv().await.unwrap();
12804 }
12805
12806 #[tokio::test(start_paused = true)]
12807 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
12808 let handler = ControlHandler::with_forwarding(
12809 Arc::new(Registry::default()),
12810 Arc::new(ForwardingTable::default()),
12811 );
12812 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
12813 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
12814 hello_via_sink(
12815 &handler,
12816 &first_ctx,
12817 &mut first_rx,
12818 hello_frame("first", PROTOCOL_VERSION, 1),
12819 )
12820 .await;
12821 hello_via_sink(
12822 &handler,
12823 &second_ctx,
12824 &mut second_rx,
12825 hello_frame("second", PROTOCOL_VERSION, 2),
12826 )
12827 .await;
12828 let root = unique_project_root("second-only");
12829 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
12830 let cloned = handler.clone();
12831 let open = tokio::spawn(async move {
12832 cloned
12833 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
12834 .await
12835 .unwrap()
12836 });
12837 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
12838 .await
12839 .unwrap()
12840 .unwrap();
12841 let first = query_live_roots(&handler, &first_ctx).await;
12842 let second = query_live_roots(&handler, &second_ctx).await;
12843 assert!(
12844 matches!(
12845 first,
12846 ModuleControlResponseToModule::LiveRoots {
12847 total_bindings: 0,
12848 ..
12849 }
12850 ),
12851 "cross-module scope must not expose another module's roots"
12852 );
12853 assert!(
12854 matches!(
12855 second,
12856 ModuleControlResponseToModule::LiveRoots {
12857 total_bindings: 1,
12858 ..
12859 }
12860 ),
12861 "second module must see its pending route"
12862 );
12863 handler
12864 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
12865 .await
12866 .unwrap();
12867 assert!(open.await.unwrap().is_empty());
12868 }
12869
12870 #[tokio::test(start_paused = true)]
12871 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
12872 let handler = ControlHandler::with_forwarding(
12873 Arc::new(Registry::default()),
12874 Arc::new(ForwardingTable::default()),
12875 );
12876 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
12877 hello_via_sink(
12878 &handler,
12879 &target_ctx,
12880 &mut target_rx,
12881 hello_frame("target", PROTOCOL_VERSION, 1),
12882 )
12883 .await;
12884 let actual = query_live_roots(&handler, &target_ctx).await;
12885 let ModuleControlResponseToModule::LiveRoots {
12886 roots,
12887 unknown_root_bindings,
12888 total_bindings,
12889 } = actual
12890 else {
12891 panic!("expected live roots")
12892 };
12893 assert!(roots.is_empty());
12894 assert_eq!(unknown_root_bindings, 0);
12895 assert_eq!(total_bindings, 0);
12896 assert_eq!(
12897 total_bindings,
12898 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12899 );
12900 }
12901
12902 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
12909 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12910 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
12911 let text = std::fs::read_to_string(&path)
12912 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
12913 let vectors: Vec<(String, Value)> = text
12914 .lines()
12915 .filter(|line| !line.trim().is_empty())
12916 .map(|line| {
12917 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
12918 let id = entry["corpus_id"]
12919 .as_str()
12920 .expect("every vector carries a corpus_id")
12921 .to_string();
12922 (id, entry["package"].clone())
12923 })
12924 .collect();
12925 assert_eq!(
12929 vectors.len(),
12930 3,
12931 "vendored fed corpus changed size; re-sync from subc-federation"
12932 );
12933
12934 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
12949 let richest = vectors
12950 .iter()
12951 .filter_map(|(_, package)| package.as_object())
12952 .map(|object| {
12953 object
12954 .keys()
12955 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
12956 .count()
12957 })
12958 .max()
12959 .unwrap_or(0);
12960 assert!(
12961 richest >= 2,
12962 "vendored corpus no longer carries a package with unmodelled fields, \
12963 so the relay test can no longer distinguish a verbatim relay from a lossy one"
12964 );
12965
12966 vectors
12967 }
12968
12969 #[tokio::test]
12977 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12978 for (corpus_id, package) in fed_admission_facts_vectors() {
12979 let registry = Arc::new(Registry::default());
12980 let forwarding = Arc::new(ForwardingTable::default());
12981 let supervisor = SupervisorHandle::new();
12982 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12983 let handler =
12984 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12985 .with_supervisor(supervisor)
12986 .with_admission_facts_config(
12987 Some("fed".to_string()),
12988 Some(vec!["target".to_string()]),
12989 );
12990
12991 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12992 hello_via_sink(
12993 &handler,
12994 &target_ctx,
12995 &mut target_rx,
12996 hello_frame("target", PROTOCOL_VERSION, 1),
12997 )
12998 .await;
12999
13000 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
13001 let route_handler = handler.clone();
13002 let expected = package.clone();
13003 let route_task = tokio::spawn(async move {
13004 route_handler
13005 .handle_control_frame(
13006 &client_ctx,
13007 route_open_frame_with_admission_facts(
13008 20,
13009 "target",
13010 unique_project_root("admission-facts"),
13011 Some(subc_control::ConsumerIdentity {
13012 module_id: "fed".to_string(),
13013 launch_nonce: "fed-nonce".to_string(),
13014 }),
13015 Some(package),
13016 ),
13017 )
13018 .await
13019 .unwrap()
13020 });
13021
13022 let bind_frame = target_rx.recv().await.unwrap();
13023 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13024 let ModuleControlRequest::RouteBind {
13025 admission_facts, ..
13026 } = bind
13027 else {
13028 panic!("{corpus_id}: expected route.bind")
13029 };
13030 assert_eq!(
13031 admission_facts,
13032 Some(expected),
13033 "{corpus_id}: relay must not add, drop or reshape any field"
13034 );
13035
13036 handler
13037 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13038 .await
13039 .unwrap();
13040 route_task.await.unwrap();
13041 }
13042 }
13043
13044 #[tokio::test]
13045 async fn route_bind_never_relays_the_openers_build_class() {
13046 let registry = Arc::new(Registry::default());
13047 let forwarding = Arc::new(ForwardingTable::default());
13048 let handler =
13049 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13050 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
13051 hello_via_sink(
13052 &handler,
13053 &target_ctx,
13054 &mut target_rx,
13055 hello_frame("target", PROTOCOL_VERSION, 1),
13056 )
13057 .await;
13058
13059 let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
13060 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
13061 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
13062 busy_refusals: None,
13063 target: RouteTarget::ToolProvider {
13064 module_id: "target".to_string(),
13065 },
13066 identity,
13067 consumer_identity: None,
13068 consumer_capabilities: None,
13069 role_versions: None,
13070 admission_facts: None,
13071 scope: None,
13072 })
13073 .unwrap();
13074 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
13075
13076 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
13077 let route_handler = handler.clone();
13078 let route_task = tokio::spawn(async move {
13079 route_handler
13080 .handle_control_frame(&client_ctx, open)
13081 .await
13082 .unwrap()
13083 });
13084 let bind_frame = target_rx.recv().await.unwrap();
13085 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13086 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
13087 panic!("expected route.bind")
13088 };
13089 assert_eq!(
13090 identity.build_class, None,
13091 "route.bind carried the build class"
13092 );
13093 assert!(
13094 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
13095 "the route.bind bytes still name build_class"
13096 );
13097 handler
13098 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13099 .await
13100 .unwrap();
13101 route_task.await.unwrap();
13102 }
13103
13104 #[tokio::test]
13105 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
13106 let registry = Arc::new(Registry::default());
13107 let forwarding = Arc::new(ForwardingTable::default());
13108 let supervisor = SupervisorHandle::new();
13109 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
13110 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
13111 let handler =
13112 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13113 .with_supervisor(supervisor)
13114 .with_admission_facts_config(
13115 Some("fed".to_string()),
13116 Some(vec!["target".to_string()]),
13117 );
13118
13119 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
13120 hello_via_sink(
13121 &handler,
13122 &target_ctx,
13123 &mut target_rx,
13124 hello_frame("target", PROTOCOL_VERSION, 1),
13125 )
13126 .await;
13127 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
13128 hello_via_sink(
13129 &handler,
13130 &other_ctx,
13131 &mut other_rx,
13132 hello_frame("other", PROTOCOL_VERSION, 2),
13133 )
13134 .await;
13135
13136 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
13137 let expected_facts = facts.clone();
13138 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
13139 let route_handler = handler.clone();
13140 let route_task = tokio::spawn(async move {
13141 route_handler
13142 .handle_control_frame(
13143 &client_ctx,
13144 route_open_frame_with_admission_facts(
13145 10,
13146 "target",
13147 unique_project_root("admission-facts"),
13148 Some(subc_control::ConsumerIdentity {
13149 module_id: "fed".to_string(),
13150 launch_nonce: "fed-nonce".to_string(),
13151 }),
13152 Some(facts.clone()),
13153 ),
13154 )
13155 .await
13156 .unwrap()
13157 });
13158 let bind_frame = target_rx.recv().await.unwrap();
13159 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13160 let ModuleControlRequest::RouteBind {
13161 admission_facts, ..
13162 } = bind
13163 else {
13164 panic!("expected route.bind")
13165 };
13166 assert_eq!(admission_facts, Some(expected_facts));
13167 handler
13168 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13169 .await
13170 .unwrap();
13171 assert!(route_task.await.unwrap().is_empty());
13172 assert!(matches!(
13173 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
13174 .unwrap(),
13175 ClientControlResponse::RouteOpen { .. }
13176 ));
13177
13178 let direct = handler
13179 .handle_control_frame(
13180 &route_ctx(ConnectionId::new(73)).0,
13181 route_open_frame_with_admission_facts(
13182 11,
13183 "target",
13184 unique_project_root("admission-facts"),
13185 None,
13186 Some(json!({"x": 1})),
13187 ),
13188 )
13189 .await
13190 .unwrap();
13191 assert_eq!(
13192 parse_error(&direct[0])["code"],
13193 "admission_facts_not_permitted"
13194 );
13195
13196 let different_reserved = handler
13197 .handle_control_frame(
13198 &route_ctx(ConnectionId::new(77)).0,
13199 route_open_frame_with_admission_facts(
13200 15,
13201 "target",
13202 unique_project_root("admission-facts"),
13203 Some(subc_control::ConsumerIdentity {
13204 module_id: "other".to_string(),
13205 launch_nonce: "other-nonce".to_string(),
13206 }),
13207 Some(json!({"x": 1})),
13208 ),
13209 )
13210 .await
13211 .unwrap();
13212 assert_eq!(
13213 parse_error(&different_reserved[0])["code"],
13214 "admission_facts_not_permitted"
13215 );
13216
13217 let other_target = handler
13218 .handle_control_frame(
13219 &route_ctx(ConnectionId::new(74)).0,
13220 route_open_frame_with_admission_facts(
13221 12,
13222 "other",
13223 unique_project_root("admission-facts"),
13224 Some(subc_control::ConsumerIdentity {
13225 module_id: "fed".to_string(),
13226 launch_nonce: "fed-nonce".to_string(),
13227 }),
13228 Some(json!({"x": 1})),
13229 ),
13230 )
13231 .await
13232 .unwrap();
13233 assert_eq!(
13234 parse_error(&other_target[0])["code"],
13235 "admission_facts_target_not_allowed"
13236 );
13237
13238 let nonexistent = handler
13239 .handle_control_frame(
13240 &route_ctx(ConnectionId::new(75)).0,
13241 route_open_frame_with_admission_facts(
13242 13,
13243 "missing",
13244 unique_project_root("admission-facts"),
13245 None,
13246 Some(json!({"x": 1})),
13247 ),
13248 )
13249 .await
13250 .unwrap();
13251 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
13252
13253 let described = handler
13254 .handle_control_frame(
13255 &route_ctx(ConnectionId::new(76)).0,
13256 Frame::build(
13257 FrameType::Request,
13258 control_flags(),
13259 0,
13260 0,
13261 14,
13262 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13263 )
13264 .unwrap(),
13265 )
13266 .await
13267 .unwrap();
13268 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13269 serde_json::from_slice(&described[0].body).unwrap()
13270 else {
13271 panic!("expected server.describe response")
13272 };
13273 assert!(capabilities
13274 .iter()
13275 .any(|cap| cap == "admission_facts_relay_v1"));
13276 }
13277
13278 #[tokio::test]
13279 async fn admission_facts_without_configured_carrier_are_rejected() {
13280 let registry = Arc::new(Registry::default());
13281 let forwarding = Arc::new(ForwardingTable::default());
13282 let handler = ControlHandler::with_forwarding(registry, forwarding);
13283 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
13284 hello_via_sink(
13285 &handler,
13286 &target_ctx,
13287 &mut target_rx,
13288 hello_frame("target", PROTOCOL_VERSION, 1),
13289 )
13290 .await;
13291
13292 let responses = handler
13293 .handle_control_frame(
13294 &route_ctx(ConnectionId::new(79)).0,
13295 route_open_frame_with_admission_facts(
13296 16,
13297 "target",
13298 unique_project_root("admission-facts"),
13299 None,
13300 Some(json!({"x": 1})),
13301 ),
13302 )
13303 .await
13304 .unwrap();
13305 assert_eq!(
13306 parse_error(&responses[0])["code"],
13307 "admission_facts_not_permitted"
13308 );
13309 }
13310
13311 #[tokio::test]
13312 async fn route_open_relays_consumer_capabilities_verbatim() {
13313 let registry = Arc::new(Registry::default());
13314 let forwarding = Arc::new(ForwardingTable::default());
13315 let handler =
13316 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13317 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
13318 hello_via_sink(
13319 &handler,
13320 &module_ctx,
13321 &mut module_rx,
13322 hello_frame("aft", PROTOCOL_VERSION, 7),
13323 )
13324 .await;
13325
13326 let expected = vec!["elicitation".to_string(), "roots".to_string()];
13327 let expected_for_request = expected.clone();
13328 let project_root = unique_project_root("consumer-capabilities-present");
13329 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
13330 let route_handler = handler.clone();
13331 let route_task = tokio::spawn(async move {
13332 route_handler
13333 .handle_control_frame(
13334 &client_ctx,
13335 route_open_frame_with_consumer_capabilities(
13336 401,
13337 "aft",
13338 project_root,
13339 Some(expected_for_request),
13340 ),
13341 )
13342 .await
13343 .unwrap()
13344 });
13345 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13346 .await
13347 .unwrap()
13348 .unwrap();
13349 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13350 let ModuleControlRequest::RouteBind {
13351 consumer_capabilities,
13352 ..
13353 } = bind
13354 else {
13355 panic!("expected route.bind request, got {bind:?}");
13356 };
13357 assert_eq!(consumer_capabilities, Some(expected.clone()));
13358
13359 handler
13360 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13361 .await
13362 .unwrap();
13363 let route_response = route_task.await.unwrap();
13364 assert!(route_response.is_empty());
13365 let published = client_rx.recv().await.unwrap();
13366 assert!(matches!(
13367 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13368 ClientControlResponse::RouteOpen { .. }
13369 ));
13370 }
13371
13372 #[tokio::test]
13373 async fn route_open_without_consumer_capabilities_relays_none() {
13374 let registry = Arc::new(Registry::default());
13375 let forwarding = Arc::new(ForwardingTable::default());
13376 let handler =
13377 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13378 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
13379 hello_via_sink(
13380 &handler,
13381 &module_ctx,
13382 &mut module_rx,
13383 hello_frame("aft", PROTOCOL_VERSION, 7),
13384 )
13385 .await;
13386
13387 let project_root = unique_project_root("consumer-capabilities-absent");
13388 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
13389 let route_handler = handler.clone();
13390 let route_task = tokio::spawn(async move {
13391 route_handler
13392 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
13393 .await
13394 .unwrap()
13395 });
13396 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13397 .await
13398 .unwrap()
13399 .unwrap();
13400 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13401 let ModuleControlRequest::RouteBind {
13402 consumer_capabilities,
13403 ..
13404 } = bind
13405 else {
13406 panic!("expected route.bind request, got {bind:?}");
13407 };
13408 assert_eq!(consumer_capabilities, None);
13409
13410 handler
13411 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13412 .await
13413 .unwrap();
13414 let route_response = route_task.await.unwrap();
13415 assert!(route_response.is_empty());
13416 let published = client_rx.recv().await.unwrap();
13417 assert!(matches!(
13418 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13419 ClientControlResponse::RouteOpen { .. }
13420 ));
13421 }
13422
13423 async fn bind_role_versions_for(
13427 sent: Option<BTreeMap<String, String>>,
13428 connection: u64,
13429 ) -> Option<BTreeMap<String, String>> {
13430 let registry = Arc::new(Registry::default());
13431 let forwarding = Arc::new(ForwardingTable::default());
13432 let handler =
13433 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13434 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
13435 hello_via_sink(
13436 &handler,
13437 &module_ctx,
13438 &mut module_rx,
13439 hello_frame("aft", PROTOCOL_VERSION, 7),
13440 )
13441 .await;
13442 let project_root = unique_project_root("role-versions");
13443 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
13444 let route_handler = handler.clone();
13445 let route_task = tokio::spawn(async move {
13446 route_handler
13447 .handle_control_frame(
13448 &client_ctx,
13449 route_open_frame_with_role_versions(403, "aft", project_root, sent),
13450 )
13451 .await
13452 .unwrap()
13453 });
13454 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13455 .await
13456 .expect("a well-formed route.open reaches the module as a bind")
13457 .unwrap();
13458 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13459 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
13460 panic!("expected route.bind request, got {bind:?}");
13461 };
13462 handler
13463 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13464 .await
13465 .unwrap();
13466 assert!(route_task.await.unwrap().is_empty());
13467 let published = client_rx.recv().await.unwrap();
13468 assert!(matches!(
13469 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13470 ClientControlResponse::RouteOpen { .. }
13471 ));
13472 role_versions
13473 }
13474
13475 #[tokio::test]
13476 async fn route_open_relays_role_versions_verbatim() {
13477 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
13478 assert_eq!(
13479 bind_role_versions_for(Some(sent.clone()), 141).await,
13480 Some(sent)
13481 );
13482 }
13483
13484 #[tokio::test]
13487 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
13488 assert_eq!(bind_role_versions_for(None, 143).await, None);
13489 assert_eq!(
13490 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
13491 None
13492 );
13493 }
13494
13495 #[tokio::test]
13496 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
13497 let registry = Arc::new(Registry::default());
13498 let forwarding = Arc::new(ForwardingTable::default());
13499 let handler =
13500 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13501 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
13502 hello_via_sink(
13503 &handler,
13504 &module_ctx,
13505 &mut module_rx,
13506 hello_frame("aft", PROTOCOL_VERSION, 7),
13507 )
13508 .await;
13509
13510 let nine: BTreeMap<String, String> = (0..9)
13511 .map(|index| (format!("role-{index}"), "v1".to_string()))
13512 .collect();
13513 for (label, malformed) in [
13514 (
13515 "invalid role name",
13516 role_versions(&[("Tool_Provider", "v1")]),
13517 ),
13518 ("invalid version", role_versions(&[("tool-provider", "v0")])),
13519 ("nine entries", nine),
13520 ] {
13521 let responses = tokio::time::timeout(
13524 Duration::from_secs(1),
13525 handler.handle_control_frame(
13526 &route_ctx(ConnectionId::new(148)).0,
13527 route_open_frame_with_role_versions(
13528 404,
13529 "aft",
13530 unique_project_root("role-versions-malformed"),
13531 Some(malformed),
13532 ),
13533 ),
13534 )
13535 .await
13536 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
13537 .unwrap();
13538 assert_eq!(responses.len(), 1, "{label}");
13539 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
13540 let error = parse_error(&responses[0]);
13541 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
13542 assert_eq!(
13543 error["detail"]["field"], "role_versions",
13544 "{label}: {error}"
13545 );
13546 assert!(
13547 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
13548 "{label}: a malformed declaration is terminal"
13549 );
13550 assert!(
13551 module_rx.try_recv().is_err(),
13552 "{label}: the module must never see a bind"
13553 );
13554 }
13555 }
13556
13557 #[tokio::test]
13561 async fn route_role_versions_capability_is_advertised() {
13562 let handler = ControlHandler::new(Arc::new(Registry::default()));
13563 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
13564 let ack = hello_via_sink(
13565 &handler,
13566 &ctx,
13567 &mut rx,
13568 hello_frame("m", PROTOCOL_VERSION, 1),
13569 )
13570 .await;
13571 let ack = parse_ack(&ack);
13572 assert!(
13573 ack.subc_capabilities
13574 .iter()
13575 .any(|c| c == "route-role-versions/v1"),
13576 "{:?}",
13577 ack.subc_capabilities
13578 );
13579
13580 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13581 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13582 let reply = handler
13583 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
13584 .await
13585 .unwrap()
13586 .pop()
13587 .unwrap();
13588 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13589 serde_json::from_slice(&reply.body).unwrap()
13590 else {
13591 panic!("not a server.describe reply");
13592 };
13593 assert!(
13594 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
13595 "{capabilities:?}"
13596 );
13597 }
13598
13599 #[tokio::test]
13603 async fn catalog_events_capability_is_advertised() {
13604 let handler = ControlHandler::new(Arc::new(Registry::default()));
13605 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13606 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13607 let reply = handler
13608 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
13609 .await
13610 .unwrap()
13611 .pop()
13612 .unwrap();
13613 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13614 serde_json::from_slice(&reply.body).unwrap()
13615 else {
13616 panic!("not a server.describe reply");
13617 };
13618 assert!(
13619 capabilities
13620 .iter()
13621 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
13622 "{capabilities:?}"
13623 );
13624 }
13625
13626 #[tokio::test]
13627 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
13628 let handler = ControlHandler::new(Arc::new(Registry::default()));
13629 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
13630 let ack = hello_via_sink(
13631 &handler,
13632 &module,
13633 &mut module_rx,
13634 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
13635 )
13636 .await;
13637 let ack = parse_ack(&ack);
13638 assert!(
13639 ack.subc_capabilities
13640 .iter()
13641 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13642 "{:?}",
13643 ack.subc_capabilities
13644 );
13645 assert!(ack
13646 .subc_capabilities
13647 .iter()
13648 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13649
13650 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
13651 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13652 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13653 let reply = handler
13654 .handle_control_frame(&client, frame)
13655 .await
13656 .unwrap()
13657 .pop()
13658 .unwrap();
13659 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13660 serde_json::from_slice(&reply.body).unwrap()
13661 else {
13662 panic!("not a server.describe reply");
13663 };
13664 assert!(
13665 capabilities
13666 .iter()
13667 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13668 "{capabilities:?}"
13669 );
13670 assert!(capabilities
13671 .iter()
13672 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13673 }
13674
13675 #[tokio::test]
13676 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
13677 let handler = ControlHandler::new(Arc::new(Registry::default()));
13678 let (module, mut rx) = route_ctx(ConnectionId::new(156));
13679 let ack = hello_via_sink(
13680 &handler,
13681 &module,
13682 &mut rx,
13683 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
13684 )
13685 .await;
13686 assert!(parse_ack(&ack)
13687 .subc_capabilities
13688 .iter()
13689 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13690 assert!(parse_ack(&ack)
13691 .subc_capabilities
13692 .iter()
13693 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13694 let (client, _rx) = route_ctx(ConnectionId::new(157));
13695 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13696 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13697 let reply = handler
13698 .handle_control_frame(&client, frame)
13699 .await
13700 .unwrap()
13701 .pop()
13702 .unwrap();
13703 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13704 serde_json::from_slice(&reply.body).unwrap()
13705 else {
13706 panic!("server.describe reply")
13707 };
13708 assert!(capabilities
13709 .iter()
13710 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13711 assert!(capabilities
13712 .iter()
13713 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13714 }
13715
13716 #[tokio::test]
13717 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
13718 let handler = ControlHandler::new(Arc::new(Registry::default()));
13719 let (module, mut rx) = route_ctx(ConnectionId::new(158));
13720 let ack = hello_via_sink(
13721 &handler,
13722 &module,
13723 &mut rx,
13724 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
13725 )
13726 .await;
13727 assert!(parse_ack(&ack)
13728 .subc_capabilities
13729 .iter()
13730 .any(|cap| cap == "scope-workdir/v1"));
13731 let (client, _rx) = route_ctx(ConnectionId::new(159));
13732 let frame = Frame::build(
13733 FrameType::Request,
13734 control_flags(),
13735 0,
13736 0,
13737 5,
13738 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13739 )
13740 .unwrap();
13741 let reply = handler
13742 .handle_control_frame(&client, frame)
13743 .await
13744 .unwrap()
13745 .remove(0);
13746 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13747 serde_json::from_slice(&reply.body).unwrap()
13748 else {
13749 panic!("server.describe reply")
13750 };
13751 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
13752 }
13753
13754 #[tokio::test]
13755 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
13756 let handler = ControlHandler::new(Arc::new(Registry::default()));
13757 let (module, mut rx) = route_ctx(ConnectionId::new(160));
13758 let ack = hello_via_sink(
13759 &handler,
13760 &module,
13761 &mut rx,
13762 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
13763 )
13764 .await;
13765 assert!(parse_ack(&ack)
13766 .subc_capabilities
13767 .iter()
13768 .any(|cap| cap == "worker-runs/v1"));
13769 let (client, _rx) = route_ctx(ConnectionId::new(161));
13770 let frame = Frame::build(
13771 FrameType::Request,
13772 control_flags(),
13773 0,
13774 0,
13775 5,
13776 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13777 )
13778 .unwrap();
13779 let reply = handler
13780 .handle_control_frame(&client, frame)
13781 .await
13782 .unwrap()
13783 .remove(0);
13784 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13785 serde_json::from_slice(&reply.body).unwrap()
13786 else {
13787 panic!("server.describe reply")
13788 };
13789 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
13790 }
13791
13792 #[tokio::test]
13793 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
13794 let handler = ControlHandler::new(Arc::new(Registry::default()));
13795 let (module, mut rx) = route_ctx(ConnectionId::new(154));
13796 let ack = hello_via_sink(
13797 &handler,
13798 &module,
13799 &mut rx,
13800 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
13801 )
13802 .await;
13803 assert!(parse_ack(&ack)
13804 .subc_capabilities
13805 .iter()
13806 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13807 let (client, _rx) = route_ctx(ConnectionId::new(155));
13808 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13809 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13810 let reply = handler
13811 .handle_control_frame(&client, frame)
13812 .await
13813 .unwrap()
13814 .pop()
13815 .unwrap();
13816 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13817 serde_json::from_slice(&reply.body).unwrap()
13818 else {
13819 panic!("not a server.describe reply")
13820 };
13821 assert!(capabilities
13822 .iter()
13823 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13824 }
13825
13826 #[tokio::test]
13827 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
13828 let registry = Arc::new(Registry::default());
13829 let forwarding = Arc::new(ForwardingTable::default());
13830 let handler =
13831 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13832 .with_health_probe_timeout(Duration::from_secs(5));
13833 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
13834 hello_via_sink(
13835 &handler,
13836 &module_ctx,
13837 &mut module_rx,
13838 non_routable_hello_frame_with_control_ops(
13839 "mcp",
13840 300,
13841 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13842 ),
13843 )
13844 .await;
13845 assert!(registry
13846 .get_module("mcp")
13847 .unwrap()
13848 .unwrap()
13849 .manifest
13850 .provides
13851 .is_empty());
13852
13853 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
13854 let route_response = handler
13855 .handle_control_frame(
13856 &route_client_ctx,
13857 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
13858 )
13859 .await
13860 .unwrap();
13861 assert_eq!(route_response[0].header.ty, FrameType::Error);
13862 assert_eq!(
13863 parse_error(&route_response[0])["code"],
13864 "target_unavailable"
13865 );
13866 assert!(parse_error(&route_response[0])["message"]
13867 .as_str()
13868 .unwrap()
13869 .contains("does not provide the requested target"));
13870 assert!(module_rx.try_recv().is_err());
13871
13872 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
13873 let health_handler = handler.clone();
13874 let health_task = tokio::spawn(async move {
13875 health_handler
13876 .handle_control_frame(
13877 &health_client_ctx,
13878 supervisor_health_probe_frame(302, "mcp"),
13879 )
13880 .await
13881 .unwrap()
13882 });
13883 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13884 .await
13885 .unwrap()
13886 .unwrap();
13887 assert_eq!(
13888 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13889 ModuleControlRequest::HealthCheck {}
13890 );
13891 handler
13892 .handle_control_frame(
13893 &module_ctx,
13894 health_response(health_frame.header.corr, HealthStatus::Ok),
13895 )
13896 .await
13897 .unwrap();
13898 let health_response = health_task.await.unwrap();
13899 assert_eq!(health_response[0].header.ty, FrameType::Response);
13900 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13901 ClientControlResponse::SupervisorHealthProbe {
13902 module_id, status, ..
13903 } => {
13904 assert_eq!(module_id, "mcp");
13905 assert_eq!(status, HealthStatus::Ok);
13906 }
13907 other => panic!("unexpected health response: {other:?}"),
13908 }
13909
13910 forwarding
13915 .cleanup_connection(module_ctx.connection_id)
13916 .unwrap();
13917 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
13918 let cleanup_response = tokio::time::timeout(
13919 Duration::from_millis(200),
13920 handler.handle_control_frame(
13921 &cleanup_probe_ctx,
13922 supervisor_health_probe_frame(303, "mcp"),
13923 ),
13924 )
13925 .await
13926 .expect("probe should fail immediately when the control lane is gone")
13927 .unwrap();
13928 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
13929 assert_eq!(
13930 parse_error(&cleanup_response[0])["code"],
13931 "target_unavailable"
13932 );
13933 assert!(parse_error(&cleanup_response[0])["message"]
13934 .as_str()
13935 .unwrap()
13936 .contains("no module connection"));
13937
13938 handler
13939 .cleanup_connection(module_ctx.connection_id)
13940 .unwrap();
13941 }
13942
13943 #[tokio::test]
13944 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
13945 let registry = Arc::new(Registry::default());
13946 let supervisor_handle = SupervisorHandle::new();
13947 let supervisor =
13948 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13949 .with_handle(supervisor_handle.clone())
13950 .with_connection_file_path(
13951 std::env::temp_dir()
13952 .join(format!("subc-route-open-warming-{}", std::process::id())),
13953 );
13954 let module = supervisor
13955 .supervise_configured(
13956 ModuleSpec {
13957 carry_containment: false,
13958 module_id: "warming".to_string(),
13959 program: fake_aft_stub_path(),
13960 args: Vec::new(),
13961 env: Vec::new(),
13962 reserved: false,
13963 reserved_prefixes: Vec::new(),
13964 protocol: ModuleProtocol::Subc,
13965 overlap: Default::default(),
13966 },
13967 true,
13968 )
13969 .unwrap();
13970 assert_eq!(module.state().unwrap(), ModuleState::Running);
13971
13972 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13973 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13974 let response = handler
13975 .handle_control_frame(
13976 &ctx,
13977 route_open_frame(304, "warming", unique_project_root("warming")),
13978 )
13979 .await
13980 .unwrap();
13981 module.stop().await.unwrap();
13982
13983 assert_eq!(response[0].header.ty, FrameType::Error);
13984 let error = parse_error(&response[0]);
13985 assert_eq!(error["code"], "module_warming");
13986 assert!(error["message"]
13987 .as_str()
13988 .unwrap()
13989 .contains("state=running, enabled=true, live=false"));
13990 }
13991
13992 #[test]
13993 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13994 let handler = ControlHandler::new(Arc::new(Registry::default()));
13995 let capture = EventCapture::default();
13996 let _subscriber =
13997 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13998 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13999 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
14000 let pending = (0..limit).collect::<Vec<_>>();
14001 let response = handler
14002 .route_open_capacity_refusal(
14003 &ctx,
14004 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
14005 "busy",
14006 pending.len(),
14007 limit,
14008 )
14009 .unwrap();
14010 assert_eq!(parse_error(&response)["code"], "target_unavailable");
14011 let event = capture
14012 .events()
14013 .into_iter()
14014 .find(|event| {
14015 event.target == "control"
14016 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
14017 })
14018 .expect("connection admission refusal event");
14019 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
14020 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
14021 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14024 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14025 }
14026
14027 #[test]
14028 fn route_open_target_cap_logs_admission_reason_and_capacity() {
14029 let handler = ControlHandler::new(Arc::new(Registry::default()));
14030 let capture = EventCapture::default();
14031 let _subscriber =
14032 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14033 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
14034 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
14035 let guards = (0..limit)
14036 .map(|_| {
14037 handler
14038 .route_bind_concurrency
14039 .try_admit("busy", limit)
14040 .unwrap()
14041 })
14042 .collect::<Vec<_>>();
14043 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
14044 Err(in_flight) => in_flight,
14045 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
14046 };
14047 let response = handler
14048 .route_open_target_capacity_refusal(
14049 &ctx,
14050 &RouteOpenCaller::unknown(),
14051 &route_open_frame(397, "busy", unique_project_root("target-cap")),
14052 "busy",
14053 in_flight,
14054 )
14055 .unwrap();
14056 assert_eq!(parse_error(&response)["code"], "target_unavailable");
14057 let event = capture
14058 .events()
14059 .into_iter()
14060 .find(|event| {
14061 event.target == "control"
14062 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
14063 })
14064 .expect("target admission refusal event");
14065 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
14066 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
14067 drop(guards);
14068 }
14069
14070 #[cfg(unix)]
14071 #[tokio::test(start_paused = true)]
14072 async fn supervisor_list_shows_a_failed_modules_next_retry() {
14073 let registry = Arc::new(Registry::default());
14074 let handle = SupervisorHandle::new();
14075 let supervisor =
14076 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
14077 .with_handle(handle.clone());
14078 let module = supervisor
14079 .spawn(crate::ModuleSpec {
14080 carry_containment: false,
14081 module_id: "retry-list".into(),
14082 program: PathBuf::from("/bin/sh"),
14083 args: vec!["-c".into(), "exit 17".into()],
14084 env: vec![],
14085 reserved: false,
14086 reserved_prefixes: vec![],
14087 protocol: ModuleProtocol::None,
14088 overlap: Default::default(),
14089 })
14090 .unwrap();
14091 let deadline = std::time::Instant::now() + Duration::from_secs(10);
14092 while module.state().unwrap() != crate::ModuleState::Failed {
14093 assert!(
14094 std::time::Instant::now() < deadline,
14095 "waiting for budget failure"
14096 );
14097 tokio::task::yield_now().await;
14098 }
14099 let handler = ControlHandler::new(registry).with_supervisor(handle);
14100 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14101 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14102 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14103 assert_eq!(body["modules"][0]["state"], "failed");
14104 let retry = body["modules"][0]["next_retry_at_ms"]
14105 .as_u64()
14106 .expect("scheduled retry on the wire");
14107 let now = std::time::SystemTime::now()
14108 .duration_since(std::time::UNIX_EPOCH)
14109 .unwrap()
14110 .as_millis() as u64;
14111 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
14112 module.set_enabled(false).await.unwrap();
14113 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14114 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14115 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14116 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
14117 }
14118
14119 #[tokio::test]
14120 async fn slow_step_before_pid_read_does_not_spend_its_budget() {
14121 let _guard = crate::subtree_resources::fault_read_guard().await;
14122 let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
14123 let supervisor =
14124 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14125 let module = supervisor
14126 .spawn(crate::ModuleSpec {
14127 carry_containment: false,
14128 module_id: "pid-read-budget".into(),
14129 program: std::env::current_exe().unwrap(),
14130 args: vec![
14131 "--exact".into(),
14132 "control::tests::pid_resource_read_fixture".into(),
14133 ],
14134 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14135 .into_iter()
14136 .map(|key| (key.into(), dir.display().to_string()))
14137 .chain(std::iter::once((
14138 "SUBC_SUBTREE_PID_FIXTURE".into(),
14139 "1".into(),
14140 )))
14141 .collect(),
14142 reserved: false,
14143 reserved_prefixes: Vec::new(),
14144 protocol: ModuleProtocol::None,
14145 overlap: Default::default(),
14146 })
14147 .unwrap();
14148 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
14149 while !matches!(
14150 module.child_resource_usage(),
14151 subc_control::ChildResourceUsage::Measured(_)
14152 ) {
14153 assert!(
14154 tokio::time::Instant::now() < ready_deadline,
14155 "fixture process was not measurable"
14156 );
14157 tokio::time::sleep(Duration::from_millis(10)).await;
14158 }
14159 tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
14160 let reading = crate::subtree_resources::read_process(module.clone()).await;
14161 module.stop().await.unwrap();
14162 assert!(
14163 matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
14164 "{reading:?}"
14165 );
14166 }
14167
14168 #[test]
14169 fn pid_resource_read_fixture() {
14170 if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
14171 loop {
14172 std::thread::park_timeout(Duration::from_secs(600));
14173 }
14174 }
14175 }
14176
14177 #[tokio::test]
14178 async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
14179 let registry = Arc::new(Registry::default());
14180 let handle = SupervisorHandle::new();
14181 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14182 .with_handle(handle.clone());
14183 supervisor
14184 .supervise_configured(
14185 crate::ModuleSpec {
14186 module_id: "disabled-resource-reading".into(),
14187 program: std::env::current_exe().unwrap(),
14188 args: Vec::new(),
14189 env: Vec::new(),
14190 reserved: false,
14191 reserved_prefixes: Vec::new(),
14192 protocol: ModuleProtocol::None,
14193 overlap: Default::default(),
14194 carry_containment: false,
14195 },
14196 false,
14197 )
14198 .unwrap();
14199 let handler = ControlHandler::new(registry).with_supervisor(handle);
14200 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14201 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14202 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14203 assert!(body["modules"][0].get("tree").is_none());
14204 assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
14205 }
14206
14207 #[tokio::test]
14208 async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
14209 let _guard = crate::subtree_resources::fault_read_guard().await;
14210 let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
14211 let registry = Arc::new(Registry::default());
14212 let handle = SupervisorHandle::new();
14213 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14214 .with_handle(handle.clone());
14215 let module = supervisor
14216 .spawn(crate::ModuleSpec {
14217 carry_containment: false,
14218 module_id: "list-process-priority".into(),
14219 program: std::env::current_exe().unwrap(),
14220 args: vec![
14221 "--exact".into(),
14222 "control::tests::pid_resource_read_fixture".into(),
14223 ],
14224 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14225 .into_iter()
14226 .map(|key| (key.into(), dir.display().to_string()))
14227 .chain(std::iter::once((
14228 "SUBC_SUBTREE_PID_FIXTURE".into(),
14229 "1".into(),
14230 )))
14231 .collect(),
14232 reserved: false,
14233 reserved_prefixes: Vec::new(),
14234 protocol: ModuleProtocol::None,
14235 overlap: Default::default(),
14236 })
14237 .unwrap();
14238 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
14239 loop {
14240 let image = module.running_image_agreement().await;
14241 if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
14242 break;
14243 }
14244 assert!(
14245 tokio::time::Instant::now() < ready_deadline,
14246 "the fixture process never became observable: {image:?}"
14247 );
14248 tokio::time::sleep(Duration::from_millis(10)).await;
14249 }
14250 let (target, release, _) = crate::subtree_resources::held();
14251 module.set_subtree_target_for_test(target);
14252 let handler = ControlHandler::new(registry).with_supervisor(handle);
14253 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14254 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14255 drop(release);
14256 module.stop().await.unwrap();
14257 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14258 assert_eq!(
14259 body["modules"][0]["resources"]["status"], "measured",
14260 "the optional tree timeout must not discard the process reading: {body}"
14261 );
14262 assert!(body["modules"][0].get("tree").is_none());
14263 assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
14264 assert_eq!(
14265 body["modules"][0]["pending_reload"]["image"]["status"], "match",
14266 "the tree timeout must not hide a completed image observation"
14267 );
14268 }
14269
14270 #[cfg(any(target_os = "linux", target_os = "macos"))]
14271 #[tokio::test]
14272 async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
14273 let mut handler = ControlHandler::new(Arc::new(Registry::default()))
14274 .with_daemon_provenance(
14275 std::process::id(),
14276 0,
14277 Some(std::env::current_exe().unwrap()),
14278 None,
14279 None,
14280 );
14281 let (probe, release, began) = ExecutableIdentityProbe::held();
14282 handler.daemon_provenance.probe = probe;
14283 let observe = |handler: ControlHandler, corr| {
14284 tokio::spawn(async move {
14285 let (ctx, _) = route_ctx(ConnectionId::new(corr));
14286 let frame =
14287 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
14288 .unwrap();
14289 handler
14290 .handle_supervisor_provenance(&ctx, frame, None)
14291 .await
14292 .unwrap()
14293 })
14294 };
14295 let explicit = observe(handler.clone(), 114);
14296 began.await.unwrap();
14297 let concurrent = observe(handler.clone(), 115);
14298 tokio::time::sleep(Duration::from_millis(300)).await;
14301 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14302 let list = tokio::time::timeout(
14303 Duration::from_secs(1),
14304 handler.handle_supervisor_list(frame),
14305 )
14306 .await;
14307 let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
14308 drop(release);
14309 let replies = explicit.await.unwrap();
14310 let concurrent_replies = concurrent.await.unwrap();
14311 assert!(
14312 explicit_was_pending,
14313 "explicit provenance must await the full observation"
14314 );
14315 let list = list
14316 .expect("a concurrent explicit probe must not stall list")
14317 .unwrap();
14318 let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
14319 assert_eq!(list_body["op"], "supervisor.list");
14320 for replies in [replies, concurrent_replies] {
14321 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14322 assert_eq!(
14323 body["daemon"]["daemon_observed"]["running_image"]["status"],
14324 "match"
14325 );
14326 }
14327 }
14328
14329 #[tokio::test]
14330 async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
14331 let _guard = crate::subtree_resources::fault_read_guard().await;
14332 let registry = Arc::new(Registry::default());
14333 let handle = SupervisorHandle::new();
14334 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14335 .with_handle(handle.clone());
14336 let module = supervisor
14337 .supervise_configured(
14338 crate::ModuleSpec {
14339 carry_containment: false,
14340 module_id: "memory-read".into(),
14341 program: PathBuf::from("disabled-module"),
14342 args: Vec::new(),
14343 env: Vec::new(),
14344 reserved: false,
14345 reserved_prefixes: Vec::new(),
14346 protocol: ModuleProtocol::Subc,
14347 overlap: Default::default(),
14348 },
14349 true,
14350 )
14351 .unwrap();
14352 let (target, release, began) = crate::subtree_resources::held();
14353 let stalled_target = target.clone();
14356 let stalled = tokio::task::spawn_blocking(move || {
14357 stalled_target.tree(&subc_os::ProcessTable::default())
14358 });
14359 began.await.unwrap();
14360 module.set_subtree_target_for_test(target);
14361 let handler = ControlHandler::new(registry).with_supervisor(handle);
14362 let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
14363 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14364 let replies = handler
14365 .handle_client_control_request(
14366 &ctx,
14367 frame,
14368 ClientControlRequest::SupervisorList {},
14369 &mut None,
14370 )
14371 .await
14372 .unwrap();
14373 assert!(
14374 replies.is_empty(),
14375 "list reply must be detached from the connection reader"
14376 );
14377 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14378 let replies = handler
14379 .handle_client_control_request(
14380 &ctx,
14381 frame,
14382 ClientControlRequest::SupervisorSpawnSnapshot {},
14383 &mut None,
14384 )
14385 .await
14386 .unwrap();
14387 assert_eq!(
14388 replies.len(),
14389 1,
14390 "another request must complete while the subtree read stalls"
14391 );
14392 let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
14393 .await
14394 .unwrap()
14395 .unwrap();
14396 release.send(()).unwrap();
14397 drop(release);
14398 stalled.await.unwrap();
14399 let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
14400 assert!(
14401 body["modules"][0].get("tree").is_none(),
14402 "an unknown tree reading must be omitted after the bounded read expires"
14403 );
14404 handler
14408 .resource_tracker
14409 .read(handler.supervisor.list(), false, "released_tree_fixture")
14410 .await;
14411 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14412 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14413 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14414 assert_eq!(
14415 body["modules"][0]["tree"]["source"], "process_tree",
14416 "the next request must export the completed refresh"
14417 );
14418 assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
14419 }
14420
14421 #[tokio::test]
14422 async fn supervisor_list_expired_budget_keeps_committed_accounting_rows() {
14423 use subc_control::{
14424 AgentCommandResources, ChildMemoryKind, ChildResourceReading,
14425 ChildResourceUnavailableReason, ChildResourceUsage, CpuLastHour, DaemonResources,
14426 TreeReading, TreeSource,
14427 };
14428 let registry = Arc::new(Registry::default());
14429 let handle = SupervisorHandle::new();
14430 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14431 .with_handle(handle.clone());
14432 supervisor
14433 .supervise_configured(
14434 crate::ModuleSpec {
14435 carry_containment: false,
14436 module_id: "budget-worker".into(),
14437 program: PathBuf::from("disabled-module"),
14438 args: Vec::new(),
14439 env: Vec::new(),
14440 reserved: false,
14441 reserved_prefixes: Vec::new(),
14442 protocol: ModuleProtocol::Subc,
14443 overlap: Default::default(),
14444 },
14445 true,
14446 )
14447 .unwrap();
14448 let handler = ControlHandler::new(registry).with_supervisor(handle);
14449 let now = u64::try_from(
14450 std::time::SystemTime::now()
14451 .duration_since(std::time::UNIX_EPOCH)
14452 .unwrap()
14453 .as_millis(),
14454 )
14455 .unwrap();
14456 let measured = now - 1_000;
14457 let hour = CpuLastHour::new(100, 60_000);
14458 let daemon = DaemonResources::new(7, 1, "test")
14459 .with_measured_at_ms(Some(measured))
14460 .with_resources(Some(ChildResourceUsage::Measured(
14461 ChildResourceReading::new(20, ChildMemoryKind::ResidentSet, 10, 0),
14462 )))
14463 .with_cpu_last_hour(Some(hour.clone()));
14464 let agents = AgentCommandResources::new("budget-worker", 50, 30, 1)
14465 .with_measured_at_ms(Some(measured))
14466 .with_cpu_last_hour(Some(hour.clone()));
14467 let tree = TreeReading::new(
14468 150,
14469 50,
14470 ChildMemoryKind::ResidentSet,
14471 1,
14472 TreeSource::ProcessTree,
14473 )
14474 .with_measured_at_ms(Some(measured));
14475 handler.resource_tracker.seed_committed_for_test(
14476 &handler.supervisor.list(),
14477 crate::resource_totals::Readings {
14478 modules: BTreeMap::from([("budget-worker".into(), (Some(tree), Some(hour)))]),
14479 daemon: Some(daemon),
14480 agent_commands: Some(vec![agents]),
14481 ..Default::default()
14482 },
14483 now,
14484 );
14485 list_diagnostics::scope(async {
14486 tokio::time::sleep(Duration::from_millis(110)).await;
14487 assert!(list_diagnostics::deadline().unwrap() <= tokio::time::Instant::now());
14488 let request =
14489 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14490 let replies = handler.handle_supervisor_list_inner(request).await.unwrap();
14491 let ClientControlResponse::SupervisorList {
14492 modules,
14493 daemon,
14494 agent_commands,
14495 ..
14496 } = serde_json::from_slice(&replies[0].body).unwrap()
14497 else {
14498 panic!("expected a list response")
14499 };
14500 assert!(matches!(
14501 modules[0].resources,
14502 Some(ChildResourceUsage::Unavailable {
14503 reason: ChildResourceUnavailableReason::Unreadable
14504 })
14505 ));
14506 assert_eq!(
14507 (
14508 modules[0].tree.as_ref().unwrap().cpu_ms,
14509 modules[0].tree.as_ref().unwrap().measured_at_ms
14510 ),
14511 (150, Some(measured))
14512 );
14513 let daemon = daemon.unwrap();
14514 let agents = agent_commands.unwrap();
14515 assert_eq!(daemon.measured_at_ms, Some(measured));
14516 assert_eq!(
14517 (agents.len(), agents[0].cpu_ms, agents[0].measured_at_ms),
14518 (1, 50, Some(measured))
14519 );
14520 for hour in [
14521 modules[0].cpu_last_hour.as_ref(),
14522 daemon.cpu_last_hour.as_ref(),
14523 agents[0].cpu_last_hour.as_ref(),
14524 ] {
14525 let hour = hour.unwrap();
14526 assert_eq!((hour.cpu_ms, hour.covered_ms), (100, 60_000));
14527 }
14528 })
14529 .await;
14530 }
14531
14532 #[tokio::test]
14533 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
14534 let registry = Arc::new(Registry::default());
14535 let handle = SupervisorHandle::new();
14536 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14537 .with_handle(handle.clone());
14538 supervisor
14539 .supervise_configured(
14540 crate::ModuleSpec {
14541 carry_containment: false,
14542 module_id: "drift".into(),
14543 program: PathBuf::from("disabled-module"),
14544 args: Vec::new(),
14545 env: Vec::new(),
14546 reserved: false,
14547 reserved_prefixes: Vec::new(),
14548 protocol: ModuleProtocol::Subc,
14549 overlap: Default::default(),
14550 },
14551 false,
14552 )
14553 .unwrap();
14554 let handler = ControlHandler::new(registry).with_supervisor(handle);
14555 for configured in [None, Some(false), Some(true)] {
14556 handler.capability_evaluator.configure(
14557 configured.map(|enabled| ("drift".to_string(), enabled)),
14558 BTreeMap::new(),
14559 );
14560 let frame =
14561 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14562 let replies = handler
14563 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
14564 .await
14565 .unwrap();
14566 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14567 assert!(body.get("configured_enabled").is_none());
14568 assert_eq!(body["op"], "supervisor.ack");
14569 let frame =
14570 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14571 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14572 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14573 assert_eq!(
14574 body["modules"][0].get("configured_enabled"),
14575 configured.map(Value::Bool).as_ref()
14576 );
14577 assert!(body["modules"][0].get("tree").is_none());
14578 assert!(body["modules"][0].get("subtree").is_none());
14579 }
14580 }
14581
14582 #[tokio::test(flavor = "current_thread")]
14583 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
14584 let handler = ControlHandler::new(Arc::new(Registry::default()));
14585 let capture = EventCapture::default();
14586 let _subscriber =
14587 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14588 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
14589 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14590
14591 let _ = handler
14592 .handle_client_control_request(
14593 &ctx,
14594 frame,
14595 ClientControlRequest::SupervisorSetEnabled {
14596 module_id: "broca".to_string(),
14597 enabled: false,
14598 },
14599 &mut None,
14600 )
14601 .await
14602 .unwrap();
14603
14604 let events = capture
14605 .events()
14606 .into_iter()
14607 .filter(|event| {
14608 event.target == "control"
14609 && event.fields.get("message").map(String::as_str)
14610 == Some("supervisor request received")
14611 })
14612 .collect::<Vec<_>>();
14613 assert_eq!(events.len(), 1, "one supervisor request log line");
14614 let event = &events[0];
14615 assert_eq!(event.level, tracing::Level::INFO);
14616 assert_eq!(
14617 event.fields.get("op"),
14618 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
14619 );
14620 assert_eq!(
14621 event.fields.get("module_id"),
14622 Some(&"\"broca\"".to_string())
14623 );
14624 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
14625 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
14626 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
14627 }
14628
14629 #[tokio::test(flavor = "current_thread")]
14630 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
14631 let registry = Arc::new(Registry::default());
14632 let module_connection = ConnectionId::new(303);
14633 registry
14634 .register_with_control_ops(
14635 manifest("aft", PROTOCOL_VERSION),
14636 PROTOCOL_VERSION,
14637 module_connection,
14638 module_baseline_control_ops(),
14639 )
14640 .unwrap();
14641 let handler = ControlHandler::new(registry);
14642 let capture = EventCapture::default();
14643 let _subscriber =
14644 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14645 let (ctx, _rx) = route_ctx(module_connection);
14646 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14647
14648 let _ = handler
14649 .handle_client_control_request(
14650 &ctx,
14651 frame,
14652 ClientControlRequest::SupervisorSetEnabled {
14653 module_id: "broca".to_string(),
14654 enabled: false,
14655 },
14656 &mut None,
14657 )
14658 .await
14659 .unwrap();
14660
14661 let callers = capture
14662 .events()
14663 .into_iter()
14664 .filter(|event| {
14665 event.fields.get("message").map(String::as_str)
14666 == Some("supervisor request received")
14667 })
14668 .map(|event| event.fields.get("caller").cloned())
14669 .collect::<Vec<_>>();
14670 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
14671 }
14672
14673 #[tokio::test(flavor = "current_thread")]
14674 async fn supervisor_list_does_not_log_request_received() {
14675 let handler = ControlHandler::new(Arc::new(Registry::default()));
14676 let capture = EventCapture::default();
14677 let _subscriber =
14678 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14679 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
14680 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14681
14682 handler
14683 .handle_client_control_request(
14684 &ctx,
14685 frame,
14686 ClientControlRequest::SupervisorList {},
14687 &mut None,
14688 )
14689 .await
14690 .unwrap();
14691
14692 assert!(capture.events().into_iter().all(|event| {
14693 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14694 }));
14695 }
14696
14697 #[tokio::test(flavor = "current_thread")]
14698 async fn supervisor_rescan_preview_does_not_log_request_received() {
14699 let handler = ControlHandler::new(Arc::new(Registry::default()));
14700 let capture = EventCapture::default();
14701 let _subscriber =
14702 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14703 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
14704 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14705
14706 handler
14707 .handle_client_control_request(
14708 &ctx,
14709 frame,
14710 ClientControlRequest::SupervisorRescan { preview: true },
14711 &mut None,
14712 )
14713 .await
14714 .unwrap();
14715
14716 assert!(capture.events().into_iter().all(|event| {
14717 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14718 }));
14719 }
14720
14721 struct ApproveOperator;
14722 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
14723 fn prompt(
14724 &self,
14725 _: &str,
14726 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
14727 ) -> crate::operator_confirm::ProviderResult {
14728 crate::operator_confirm::ProviderResult::Approved
14729 }
14730 }
14731
14732 #[tokio::test]
14733 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
14734 assert_eq!(
14735 ControlHandler::default().privileged_operator_prompt,
14736 PrivilegedOperatorPrompt::Log
14737 );
14738 for (initial, changed, running) in [
14739 ("log", "enforce", PrivilegedOperatorPrompt::Log),
14740 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
14741 ] {
14742 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
14743 let path = dir.join("subc.jsonc");
14744 std::fs::write(
14745 &path,
14746 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
14747 )
14748 .unwrap();
14749 let config = crate::daemon_config::load(&path).unwrap().unwrap();
14750 let registry = Arc::new(Registry::default());
14751 let handle = SupervisorHandle::new();
14752 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
14753 .with_handle(handle.clone());
14754 let handler = ControlHandler::new(registry)
14755 .with_supervisor(handle)
14756 .with_privileged_operator_prompt(config.privileged_operator_prompt)
14757 .with_supervisor_rescan(supervisor, &path, config.port);
14758 handler.forwarding.operator_confirms().configure(
14759 Arc::new(ApproveOperator),
14760 crate::operator_confirm::OperatorLimits::default(),
14761 );
14762 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
14763
14764 for (disk_mode, needs_restart) in [
14765 (Some(initial), false),
14766 (Some(changed), true),
14767 (Some(changed), true),
14768 (None, running == PrivilegedOperatorPrompt::Enforce),
14769 (Some(initial), false),
14770 ] {
14771 let mut doc = json!({"version":1});
14772 if let Some(mode) = disk_mode {
14773 doc["privileged_operator_prompt"] = json!(mode);
14774 }
14775 std::fs::write(&path, doc.to_string()).unwrap();
14776 for preview in [true, false] {
14777 let request = ClientControlRequest::SupervisorRescan { preview };
14778 let frame = Frame::build(
14779 FrameType::Request,
14780 control_flags(),
14781 0,
14782 0,
14783 1,
14784 serde_json::to_vec(&request).unwrap(),
14785 )
14786 .unwrap();
14787 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14788 assert_eq!(frames.len(), 1);
14789 let ClientControlResponse::SupervisorRescan { result } =
14790 serde_json::from_slice(&frames[0].body).unwrap()
14791 else {
14792 panic!("expected a rescan response");
14793 };
14794 let expected = if needs_restart {
14795 vec!["privileged_operator_prompt"]
14796 } else {
14797 Vec::new()
14798 };
14799 assert_eq!(result.restart_required, expected);
14800 assert_eq!(handler.privileged_operator_prompt, running);
14801 assert_eq!(
14802 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
14803 running
14804 );
14805 }
14806 }
14807 }
14808 }
14809
14810 #[tokio::test]
14811 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
14812 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
14813 let path = dir.join("subc.jsonc");
14814 let registry = Arc::new(Registry::default());
14815 let handle = SupervisorHandle::new();
14816 let supervisor =
14817 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
14818 let handler = ControlHandler::new(registry)
14819 .with_supervisor(handle)
14820 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
14821 .with_supervisor_rescan(supervisor, &path, None);
14822 handler.forwarding.operator_confirms().configure(
14823 Arc::new(ApproveOperator),
14824 crate::operator_confirm::OperatorLimits::default(),
14825 );
14826 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
14827 for preview in [true, false] {
14828 std::fs::write(
14829 &path,
14830 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
14831 )
14832 .unwrap();
14833 let request = ClientControlRequest::SupervisorRescan { preview };
14834 let frame = Frame::build(
14835 FrameType::Request,
14836 control_flags(),
14837 0,
14838 0,
14839 1,
14840 serde_json::to_vec(&request).unwrap(),
14841 )
14842 .unwrap();
14843 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14844 assert_eq!(frames.len(), 1);
14845 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
14846 assert_eq!(
14847 handler.privileged_operator_prompt,
14848 PrivilegedOperatorPrompt::Enforce
14849 );
14850 }
14851 }
14852
14853 #[tokio::test]
14858 async fn route_open_refusal_names_the_check_that_refused() {
14859 let handler = ControlHandler::new(Arc::new(Registry::default()));
14860 let capture = EventCapture::default();
14861 let _subscriber =
14862 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14863 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14864 let response = handler
14865 .handle_control_frame(
14866 &ctx,
14867 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
14868 )
14869 .await
14870 .unwrap();
14871
14872 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14873 let event = capture
14874 .events()
14875 .into_iter()
14876 .find(|event| {
14877 event.target == "control"
14878 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14879 })
14880 .expect("route.open refusal event");
14881 assert_eq!(
14882 event.fields.get("reason"),
14883 Some(&"\"not_registered\"".to_string())
14884 );
14885 }
14886
14887 #[tokio::test]
14888 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
14889 let registry = Arc::new(Registry::default());
14890 let supervisor_handle = SupervisorHandle::new();
14891 let supervisor =
14892 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
14893 .with_handle(supervisor_handle.clone())
14894 .with_connection_file_path(std::env::temp_dir().join(format!(
14895 "subc-route-open-refusal-info-{}",
14896 std::process::id()
14897 )));
14898 let module = supervisor
14899 .supervise_configured(
14900 ModuleSpec {
14901 carry_containment: false,
14902 module_id: "warming".to_string(),
14903 program: fake_aft_stub_path(),
14904 args: Vec::new(),
14905 env: Vec::new(),
14906 reserved: false,
14907 reserved_prefixes: Vec::new(),
14908 protocol: ModuleProtocol::Subc,
14909 overlap: Default::default(),
14910 },
14911 true,
14912 )
14913 .unwrap();
14914 assert_eq!(module.state().unwrap(), ModuleState::Running);
14915
14916 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14917 assert!(handler
14918 .counters()
14919 .snapshot()
14920 .get("route_open_refused_by_code")
14921 .is_none());
14922 let capture = EventCapture::default();
14923 let _subscriber =
14924 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14925 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
14926 let response = handler
14927 .handle_control_frame(
14928 &ctx,
14929 route_open_frame(394, "warming", unique_project_root("refusal-info")),
14930 )
14931 .await
14932 .unwrap();
14933 module.stop().await.unwrap();
14934
14935 assert_eq!(parse_error(&response[0])["code"], "module_warming");
14936 let event = capture
14937 .events()
14938 .into_iter()
14939 .find(|event| {
14940 event.target == "control"
14941 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
14942 })
14943 .expect("route.open refusal event");
14944 assert_eq!(
14945 event.fields.get("module_id"),
14946 Some(&"\"warming\"".to_string())
14947 );
14948 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
14949 assert_eq!(
14950 event.fields.get("reason"),
14951 Some(&"\"supervised_not_registered\"".to_string())
14952 );
14953 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
14954 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
14955 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
14956 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14957 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14958 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
14959 assert_eq!(
14960 handler.counters().snapshot()["route_open_refused_by_code"],
14961 json!({ "module_warming": 1 })
14962 );
14963 }
14964
14965 const OUTAGE_START: &str = "route.open refusing module: not serving";
14966 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
14967
14968 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
14969 capture
14970 .events()
14971 .into_iter()
14972 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
14973 .collect()
14974 }
14975
14976 fn supervise_stub(
14977 registry: &Arc<Registry>,
14978 module_id: &str,
14979 enabled: bool,
14980 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
14981 let supervisor_handle = SupervisorHandle::new();
14982 let supervisor =
14983 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
14984 .with_handle(supervisor_handle.clone())
14985 .with_connection_file_path(std::env::temp_dir().join(format!(
14986 "subc-route-outage-{module_id}-{}",
14987 std::process::id()
14988 )));
14989 let module = supervisor
14990 .supervise_configured(
14991 ModuleSpec {
14992 carry_containment: false,
14993 module_id: module_id.to_string(),
14994 program: fake_aft_stub_path(),
14995 args: Vec::new(),
14996 env: Vec::new(),
14997 reserved: false,
14998 reserved_prefixes: Vec::new(),
14999 protocol: ModuleProtocol::Subc,
15000 overlap: Default::default(),
15001 },
15002 enabled,
15003 )
15004 .unwrap();
15005 (supervisor_handle, module)
15006 }
15007
15008 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
15009 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
15010 module_id: module_id.to_string(),
15011 drain_timeout_ms: Some(50),
15012 })
15013 .unwrap();
15014 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
15015 }
15016
15017 #[test]
15021 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
15022 let registry = Arc::new(Registry::default());
15023 let forwarding = Arc::new(ForwardingTable::default());
15024 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15025 let second = ControlHandler::with_forwarding(registry, forwarding);
15026 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
15027 }
15028
15029 #[tokio::test(flavor = "current_thread")]
15033 async fn route_open_unknown_module_refusals_add_no_outage_state() {
15034 let handler = ControlHandler::new(Arc::new(Registry::default()));
15035 let capture = EventCapture::default();
15036 let _subscriber =
15037 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15038 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
15039 for corr in 0..8 {
15040 let response = handler
15041 .handle_control_frame(
15042 &ctx,
15043 route_open_frame(
15044 380 + corr,
15045 &format!("nobody-{corr}"),
15046 unique_project_root("outage-unknown"),
15047 ),
15048 )
15049 .await
15050 .unwrap();
15051 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15052 }
15053
15054 assert_eq!(handler.route_outages.tracked_module_count(), 0);
15055 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
15056 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
15057 }
15058
15059 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15064 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
15065 let registry = Arc::new(Registry::default());
15066 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
15067 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15068 let capture = EventCapture::default();
15069 let _subscriber =
15070 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15071 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
15072 handler.route_outages.record_accepted("outage-restart");
15075
15076 let response = handler
15077 .handle_control_frame(
15078 &ctx,
15079 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
15080 )
15081 .await
15082 .unwrap();
15083 assert_eq!(response[0].header.ty, FrameType::Error);
15084 let starts = outage_lines(&capture, OUTAGE_START);
15085 assert_eq!(starts.len(), 1, "{starts:?}");
15086 assert_eq!(starts[0].level, tracing::Level::WARN);
15087 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
15088 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
15089 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
15090 handler.route_outages.record_accepted("outage-restart");
15091 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
15092
15093 let restart = handler
15094 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
15095 .await
15096 .unwrap();
15097 assert_eq!(
15098 restart[0].header.ty,
15099 FrameType::Response,
15100 "{:?}",
15101 parse_error(&restart[0])
15102 );
15103 handler
15104 .handle_control_frame(
15105 &ctx,
15106 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
15107 )
15108 .await
15109 .unwrap();
15110 module.stop().await.unwrap();
15111
15112 let starts = outage_lines(&capture, OUTAGE_START);
15113 assert_eq!(starts.len(), 2, "{starts:?}");
15114 assert_eq!(starts[1].level, tracing::Level::INFO);
15115 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
15116 }
15117
15118 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15122 async fn failed_operator_restart_leaves_no_operator_mark() {
15123 let registry = Arc::new(Registry::default());
15124 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
15125 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15126 let capture = EventCapture::default();
15127 let _subscriber =
15128 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15129 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
15130 handler.route_outages.record_accepted("outage-disabled");
15131
15132 let restart = handler
15133 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
15134 .await
15135 .unwrap();
15136 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
15137 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
15138
15139 handler
15140 .handle_control_frame(
15141 &ctx,
15142 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
15143 )
15144 .await
15145 .unwrap();
15146 let starts = outage_lines(&capture, OUTAGE_START);
15147 assert_eq!(starts.len(), 1, "{starts:?}");
15148 assert_eq!(starts[0].level, tracing::Level::WARN);
15149 }
15150
15151 #[tokio::test(flavor = "current_thread")]
15152 async fn route_open_unknown_module_escapes_target_module_id() {
15153 let handler = ControlHandler::new(Arc::new(Registry::default()));
15154 let capture = EventCapture::default();
15155 let _subscriber =
15156 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15157 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
15158 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15159 let response = handler
15160 .handle_control_frame(
15161 &ctx,
15162 route_open_frame(
15163 395,
15164 hostile_module_id,
15165 unique_project_root("hostile-target-module-id"),
15166 ),
15167 )
15168 .await
15169 .unwrap();
15170
15171 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15172 let event = capture
15173 .events()
15174 .into_iter()
15175 .find(|event| {
15176 event.target == "control"
15177 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
15178 })
15179 .expect("route.open unknown-module refusal event");
15180 let logged = event.fields.get("module_id").expect("module_id field");
15181 assert!(!logged.bytes().any(|byte| byte < 0x20));
15182 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
15183 }
15184
15185 #[tokio::test(flavor = "current_thread")]
15188 async fn route_open_refusal_carries_daemon_origin() {
15189 let handler = ControlHandler::new(Arc::new(Registry::default()));
15190 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15191 let response = handler
15192 .handle_control_frame(
15193 &ctx,
15194 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
15195 )
15196 .await
15197 .unwrap();
15198
15199 assert_eq!(response.len(), 1);
15200 assert_eq!(response[0].header.ty, FrameType::Error);
15201 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15202 assert!(
15203 response[0].header.flags.is_daemon_origin(),
15204 "route.open refusal must carry daemon origin: {:?}",
15205 response[0].header.flags
15206 );
15207 }
15208
15209 #[tokio::test(flavor = "current_thread")]
15213 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
15214 let registry = Arc::new(Registry::default());
15215 let forwarding = Arc::new(ForwardingTable::default());
15216 let handler =
15217 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15218 let module_connection = ConnectionId::new(95);
15219 let (module_ctx, mut module_rx) = route_ctx(module_connection);
15220 hello_via_sink(
15221 &handler,
15222 &module_ctx,
15223 &mut module_rx,
15224 hello_frame("aft", PROTOCOL_VERSION, 395),
15225 )
15226 .await;
15227 forwarding
15228 .begin_module_drain("aft", RouteCloseReason::Restart)
15229 .unwrap()
15230 .expect("the module endpoint drains");
15231
15232 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
15233 let response = handler
15234 .handle_control_frame(
15235 &client_ctx,
15236 route_open_frame(396, "aft", unique_project_root("drain-origin")),
15237 )
15238 .await
15239 .unwrap();
15240
15241 assert_eq!(response.len(), 1);
15242 assert_eq!(response[0].header.ty, FrameType::Error);
15243 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15244 assert!(
15245 response[0].header.flags.is_daemon_origin(),
15246 "drain module_reloading must carry daemon origin: {:?}",
15247 response[0].header.flags
15248 );
15249 }
15250
15251 #[tokio::test(flavor = "current_thread")]
15255 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
15256 let registry = Arc::new(Registry::default());
15257 let forwarding = Arc::new(ForwardingTable::default());
15258 let handler =
15259 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15260 let module_connection = ConnectionId::new(95);
15261 let (module_ctx, mut module_rx) = route_ctx(module_connection);
15262 hello_via_sink(
15263 &handler,
15264 &module_ctx,
15265 &mut module_rx,
15266 hello_frame("aft", PROTOCOL_VERSION, 395),
15267 )
15268 .await;
15269
15270 let client_connection = ConnectionId::new(96);
15271 let (client_ctx, _client_rx) = route_ctx(client_connection);
15272 let (route_task, bind) = relay_route_open(
15273 &handler,
15274 client_connection,
15275 &client_ctx.egress,
15276 &mut module_rx,
15277 396,
15278 "aft",
15279 "spoofed-reloading",
15280 )
15281 .await;
15282 let rejection = Frame::build(
15285 FrameType::Error,
15286 crate::router::daemon_error_flags(),
15287 0,
15288 0,
15289 bind.header.corr,
15290 serde_json::to_vec(&ErrorBody::new(
15291 "module_reloading",
15292 "module refused route.bind",
15293 ))
15294 .unwrap(),
15295 )
15296 .unwrap();
15297 handler
15298 .handle_control_frame(&module_ctx, rejection)
15299 .await
15300 .unwrap();
15301
15302 let response = route_task.await.unwrap();
15303 assert_eq!(response.len(), 1);
15304 assert_eq!(response[0].header.ty, FrameType::Error);
15305 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15306 assert!(
15307 !response[0].header.flags.is_daemon_origin(),
15308 "a module's own rejection must not carry daemon origin: {:?}",
15309 response[0].header.flags
15310 );
15311 }
15312
15313 #[tokio::test(flavor = "current_thread")]
15314 async fn route_open_module_rejection_uses_daemon_counter_key() {
15315 let registry = Arc::new(Registry::default());
15316 let forwarding = Arc::new(ForwardingTable::default());
15317 let handler =
15318 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15319 let module_connection = ConnectionId::new(95);
15320 let (module_ctx, mut module_rx) = route_ctx(module_connection);
15321 hello_via_sink(
15322 &handler,
15323 &module_ctx,
15324 &mut module_rx,
15325 hello_frame("aft", PROTOCOL_VERSION, 395),
15326 )
15327 .await;
15328
15329 let client_connection = ConnectionId::new(96);
15330 let (client_ctx, _client_rx) = route_ctx(client_connection);
15331 let capture = EventCapture::default();
15332 let _subscriber =
15333 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15334 let (route_task, bind) = relay_route_open(
15335 &handler,
15336 client_connection,
15337 &client_ctx.egress,
15338 &mut module_rx,
15339 396,
15340 "aft",
15341 "hostile-module-code",
15342 )
15343 .await;
15344 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
15345 let rejection = Frame::build(
15346 FrameType::Error,
15347 control_flags(),
15348 0,
15349 0,
15350 bind.header.corr,
15351 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
15352 )
15353 .unwrap();
15354 handler
15355 .handle_control_frame(&module_ctx, rejection)
15356 .await
15357 .unwrap();
15358
15359 let response = route_task.await.unwrap();
15360 assert_eq!(parse_error(&response[0])["code"], hostile_code);
15361 let counters = handler.counters().snapshot();
15362 assert_eq!(
15363 counters["route_open_refused_by_code"],
15364 json!({ "module_rejected": 1 })
15365 );
15366 assert!(counters["route_open_refused_by_code"]
15367 .get(hostile_code)
15368 .is_none());
15369
15370 let event = capture
15371 .events()
15372 .into_iter()
15373 .find(|event| {
15374 event.target == "control"
15375 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
15376 })
15377 .expect("route.open module-rejection refusal event");
15378 let logged = event.fields.get("module_code").expect("module_code field");
15379 assert!(!logged.bytes().any(|byte| byte < 0x20));
15380 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
15381 }
15382
15383 #[tokio::test]
15384 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
15385 let registry = Arc::new(Registry::default());
15386 let supervisor_handle = SupervisorHandle::new();
15387 let missing_program = std::env::temp_dir().join(format!(
15388 "subc-route-open-missing-program-{}",
15389 std::process::id()
15390 ));
15391 let supervisor =
15392 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
15393 .with_handle(supervisor_handle.clone());
15394 let module = supervisor
15395 .supervise_configured(
15396 ModuleSpec {
15397 carry_containment: false,
15398 module_id: "failed".to_string(),
15399 program: missing_program,
15400 args: Vec::new(),
15401 env: Vec::new(),
15402 reserved: false,
15403 reserved_prefixes: Vec::new(),
15404 protocol: ModuleProtocol::Subc,
15405 overlap: Default::default(),
15406 },
15407 true,
15408 )
15409 .unwrap();
15410 assert_eq!(module.state().unwrap(), ModuleState::Failed);
15411
15412 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15413 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
15414 let response = handler
15415 .handle_control_frame(
15416 &ctx,
15417 route_open_frame(305, "failed", unique_project_root("failed")),
15418 )
15419 .await
15420 .unwrap();
15421
15422 assert_eq!(response[0].header.ty, FrameType::Error);
15423 let error = parse_error(&response[0]);
15424 assert_eq!(error["code"], "target_unavailable");
15425 assert!(error["message"]
15426 .as_str()
15427 .unwrap()
15428 .contains("state=failed, enabled=true, live=false"));
15429 }
15430
15431 #[tokio::test]
15432 async fn route_open_role_mismatch_remains_target_unavailable() {
15433 let registry = Arc::new(Registry::default());
15434 let handler = ControlHandler::new(Arc::clone(®istry));
15435 handler
15436 .handle_control(
15437 ConnectionId::new(41),
15438 non_routable_hello_frame_with_control_ops("health-only", 306, None),
15439 )
15440 .unwrap();
15441
15442 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
15443 let response = handler
15444 .handle_control_frame(
15445 &ctx,
15446 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
15447 )
15448 .await
15449 .unwrap();
15450
15451 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
15452 assert!(parse_error(&response[0])["message"]
15453 .as_str()
15454 .unwrap()
15455 .contains("does not provide the requested target"));
15456 }
15457
15458 #[tokio::test]
15459 async fn route_open_inactive_registration_remains_target_unavailable() {
15460 let registry = Arc::new(Registry::default());
15461 let handler = ControlHandler::new(Arc::clone(®istry));
15462 handler
15463 .handle_control(
15464 ConnectionId::new(43),
15465 hello_frame("inactive", PROTOCOL_VERSION, 308),
15466 )
15467 .unwrap();
15468 assert!(registry
15469 .set_module_state_for_test("inactive", ChannelState::Closed)
15470 .unwrap());
15471
15472 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
15473 let response = handler
15474 .handle_control_frame(
15475 &ctx,
15476 route_open_frame(309, "inactive", unique_project_root("inactive")),
15477 )
15478 .await
15479 .unwrap();
15480
15481 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
15482 assert!(parse_error(&response[0])["message"]
15483 .as_str()
15484 .unwrap()
15485 .contains("is not active"));
15486 }
15487
15488 #[tokio::test]
15489 async fn late_health_reply_is_recorded_through_the_module_response_path() {
15490 let registry = Arc::new(Registry::default());
15491 let forwarding = Arc::new(ForwardingTable::default());
15492 let supervisor_handle = SupervisorHandle::new();
15493 let supervisor =
15494 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
15495 .with_forwarding(Arc::clone(&forwarding))
15496 .with_handle(supervisor_handle.clone());
15497 let module = supervisor
15498 .supervise_configured(
15499 crate::ModuleSpec {
15500 carry_containment: false,
15501 module_id: "late-health-response".to_string(),
15502 program: PathBuf::from("disabled-module"),
15503 args: Vec::new(),
15504 env: Vec::new(),
15505 reserved: false,
15506 reserved_prefixes: Vec::new(),
15507 protocol: ModuleProtocol::Subc,
15508 overlap: Default::default(),
15509 },
15510 false,
15511 )
15512 .unwrap();
15513 let handler =
15514 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15515 .with_supervisor(supervisor_handle);
15516 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
15517 handler
15518 .handle_control_frame(
15519 &module_ctx,
15520 hello_frame_with_control_ops(
15521 "late-health-response",
15522 PROTOCOL_VERSION,
15523 7,
15524 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15525 ),
15526 )
15527 .await
15528 .unwrap();
15529 let probe_started_at = Instant::now() - Duration::from_millis(80);
15530 let pending = forwarding
15531 .begin_health_probe_rpc_for(
15532 "late-health-response",
15533 MODULE_CONTROL_OP_HEALTH_CHECK,
15534 probe_started_at,
15535 Instant::now() - Duration::from_millis(1),
15536 )
15537 .unwrap();
15538 assert!(forwarding
15539 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
15540 .unwrap());
15541
15542 let responses = handler
15543 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
15544 .await
15545 .unwrap();
15546
15547 assert!(responses.is_empty());
15548 let health = module.status().unwrap().health;
15549 assert_eq!(health.late_answer_count, 1);
15550 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
15551 }
15552
15553 #[tokio::test]
15554 async fn health_probe_timeout_and_module_death_are_typed() {
15555 let registry = Arc::new(Registry::default());
15556 let forwarding = Arc::new(ForwardingTable::default());
15557 let handler =
15558 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15559 .with_health_probe_timeout(Duration::from_millis(50));
15560 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
15561 hello_via_sink(
15562 &handler,
15563 &module_ctx,
15564 &mut module_rx,
15565 hello_frame_with_control_ops(
15566 "aft",
15567 PROTOCOL_VERSION,
15568 7,
15569 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15570 ),
15571 )
15572 .await;
15573
15574 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
15575 let responses = handler
15576 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
15577 .await
15578 .unwrap();
15579 assert_eq!(responses[0].header.ty, FrameType::Error);
15580 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
15581 let _ = module_rx.try_recv();
15582
15583 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
15584 let health_handler = handler.clone();
15585 let death_task = tokio::spawn(async move {
15586 health_handler
15587 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
15588 .await
15589 .unwrap()
15590 });
15591 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15592 .await
15593 .unwrap()
15594 .unwrap();
15595 handler
15596 .cleanup_connection(module_ctx.connection_id)
15597 .unwrap();
15598 let responses = death_task.await.unwrap();
15599 assert_eq!(responses[0].header.ty, FrameType::Error);
15600 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
15601 }
15602
15603 #[test]
15604 fn hello_requires_exact_protocol_version() {
15605 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
15606 let registry = Arc::new(Registry::default());
15607 let handler = ControlHandler::new(Arc::clone(®istry));
15608 let responses = handler
15609 .handle_control(
15610 ConnectionId::new(connection),
15611 hello_frame("aft", offered, 9),
15612 )
15613 .unwrap();
15614
15615 assert_eq!(responses.len(), 1);
15616 assert_eq!(responses[0].header.ty, FrameType::Error);
15617 let error = parse_error(&responses[0]);
15618 assert_eq!(error["code"], "version_unsupported");
15619 assert!(registry.get_module("aft").unwrap().is_none());
15620 assert_eq!(registry.active_registration_count().unwrap(), 0);
15621 }
15622 }
15623
15624 #[test]
15625 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
15626 let registry = Arc::new(Registry::default());
15627 let forwarding = Arc::new(ForwardingTable::default());
15628 let handler =
15629 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15630 let module_connection = ConnectionId::new(301);
15631 let registration = registry
15632 .register_with_control_ops(
15633 manifest("aft-push", PROTOCOL_VERSION),
15634 PROTOCOL_VERSION,
15635 module_connection,
15636 module_baseline_control_ops(),
15637 )
15638 .unwrap();
15639 let (module_tx, _module_rx) = mpsc::channel(8);
15640 let endpoint = forwarding
15641 .register_module_connection(
15642 module_connection,
15643 "aft-push".to_string(),
15644 PROTOCOL_VERSION,
15645 manifest_concurrency(®istration.manifest),
15646 FrameSink::new(module_tx),
15647 )
15648 .unwrap();
15649
15650 let unknown = Frame::build(
15652 FrameType::Push,
15653 control_flags(),
15654 0,
15655 0,
15656 5,
15657 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
15658 )
15659 .unwrap();
15660 let out = handler.handle_status_update(endpoint, unknown).unwrap();
15661 assert!(
15662 out.is_empty(),
15663 "unknown push op must be ignored, got {out:?}"
15664 );
15665
15666 let malformed = Frame::build(
15668 FrameType::Push,
15669 control_flags(),
15670 0,
15671 0,
15672 6,
15673 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
15674 )
15675 .unwrap();
15676 let out = handler.handle_status_update(endpoint, malformed).unwrap();
15677 assert_eq!(out.len(), 1);
15678 assert_eq!(out[0].header.ty, FrameType::Error);
15679 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
15680 }
15681
15682 #[test]
15683 fn hello_rejected_when_connection_already_owns_client_routes() {
15684 let registry = Arc::new(Registry::default());
15685 let forwarding = Arc::new(ForwardingTable::default());
15686 let handler =
15687 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15688 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
15690 let client_connection = ConnectionId::new(202);
15691
15692 let responses = handler
15695 .handle_control(
15696 client_connection,
15697 hello_frame("aft-second", PROTOCOL_VERSION, 9),
15698 )
15699 .unwrap();
15700 assert_eq!(responses[0].header.ty, FrameType::Error);
15701 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
15702 assert!(registry.get_module("aft-second").unwrap().is_none());
15703 }
15704
15705 #[tokio::test]
15706 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
15707 let registry = Arc::new(Registry::default());
15708 let forwarding = Arc::new(ForwardingTable::default());
15709 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
15710 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
15711 hello_via_sink(
15712 &handler,
15713 &module_ctx,
15714 &mut module_rx,
15715 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
15716 )
15717 .await;
15718 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
15719 let pending = forwarding
15720 .begin_route_bind_relay_for_test(
15721 client_ctx.connection_id,
15722 client_ctx.egress.clone(),
15723 2,
15724 "alpha",
15725 )
15726 .unwrap();
15727 forwarding
15728 .complete_pending_relay(
15729 module_ctx.connection_id,
15730 pending.corr,
15731 RouteBindRelayOutcome::Accepted,
15732 )
15733 .unwrap();
15734 client_rx.try_recv().unwrap();
15735 for module_id in ["beta", "alpha"] {
15736 let replies = handler
15737 .handle_control_frame(
15738 &module_ctx,
15739 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
15740 )
15741 .await
15742 .unwrap();
15743 assert_eq!(replies.len(), 1, "second HELLO must be refused");
15744 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
15745 }
15746 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
15747 assert!(registry.get_module("beta").unwrap().is_none());
15748 assert!(matches!(
15749 forwarding
15750 .lookup_data_route(
15751 client_ctx.connection_id,
15752 pending.client_channel,
15753 pending.client_epoch,
15754 )
15755 .unwrap(),
15756 DataRoute::Client(DataRouteState::Bound(_))
15757 ));
15758 assert!(handler
15759 .hello_launch_nonces
15760 .lock()
15761 .unwrap()
15762 .presented(module_ctx.connection_id, Some("alpha-nonce")));
15763 assert!(module_rx.try_recv().is_err());
15764 }
15765
15766 #[test]
15767 fn reserved_module_hello_requires_matching_launch_nonce() {
15768 let registry = Arc::new(Registry::default());
15769 let supervisor = SupervisorHandle::new();
15770 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
15773 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15774
15775 let no_nonce = handler
15777 .handle_control(
15778 ConnectionId::new(1),
15779 hello_frame("vault", PROTOCOL_VERSION, 1),
15780 )
15781 .unwrap();
15782 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
15783 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
15784 assert!(registry.get_module("vault").unwrap().is_none());
15785
15786 let wrong = handler
15788 .handle_control(
15789 ConnectionId::new(2),
15790 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
15791 )
15792 .unwrap();
15793 assert_eq!(wrong[0].header.ty, FrameType::Error);
15794 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
15795 assert!(registry.get_module("vault").unwrap().is_none());
15796
15797 let ok = handler
15799 .handle_control(
15800 ConnectionId::new(3),
15801 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
15802 )
15803 .unwrap();
15804 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
15805 assert!(registry.get_module("vault").unwrap().is_some());
15806 }
15807
15808 #[test]
15809 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
15810 let registry = Arc::new(Registry::default());
15811 let supervisor = SupervisorHandle::new();
15812 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15813 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15814 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15815
15816 let squat = handler
15817 .handle_control(
15818 ConnectionId::new(1),
15819 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
15820 )
15821 .unwrap();
15822 assert_eq!(squat[0].header.ty, FrameType::Error);
15823 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
15824 assert!(parse_error(&squat[0])["message"]
15825 .as_str()
15826 .unwrap()
15827 .contains("fed:"));
15828
15829 let accepted_peer = handler
15830 .handle_control(
15831 ConnectionId::new(2),
15832 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
15833 )
15834 .unwrap();
15835 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
15836
15837 let accepted_short = handler
15838 .handle_control(
15839 ConnectionId::new(3),
15840 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
15841 )
15842 .unwrap();
15843 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
15844
15845 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
15846 let response = handler
15847 .handle_control(
15848 ConnectionId::new(conn),
15849 hello_frame(module_id, PROTOCOL_VERSION, conn),
15850 )
15851 .unwrap();
15852 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
15853 }
15854 }
15855
15856 #[test]
15857 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
15858 let registry = Arc::new(Registry::default());
15859 let supervisor = SupervisorHandle::new();
15860 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15861 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15862 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
15863 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15864
15865 let owner_nonce = handler
15866 .handle_control(
15867 ConnectionId::new(1),
15868 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
15869 )
15870 .unwrap();
15871 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
15872 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
15873 assert!(registry.get_module("fed:special").unwrap().is_none());
15874
15875 let exact_nonce = handler
15876 .handle_control(
15877 ConnectionId::new(2),
15878 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
15879 )
15880 .unwrap();
15881 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
15882 assert!(registry.get_module("fed:special").unwrap().is_some());
15883 }
15884
15885 #[test]
15886 fn non_reserved_module_ignores_launch_nonce() {
15887 let registry = Arc::new(Registry::default());
15888 let handler = ControlHandler::new(Arc::clone(®istry));
15891 let no_nonce = handler
15892 .handle_control(
15893 ConnectionId::new(1),
15894 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
15895 )
15896 .unwrap();
15897 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
15898 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
15899
15900 let echoed_nonce = handler
15901 .handle_control(
15902 ConnectionId::new(2),
15903 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
15904 )
15905 .unwrap();
15906 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
15907 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
15908 }
15909
15910 #[test]
15911 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
15912 let handler = ControlHandler::default();
15913 let conn = ConnectionId::new(1);
15914 let malformed = Frame::build(
15915 FrameType::Hello,
15916 control_flags(),
15917 0,
15918 0,
15919 3,
15920 b"{not json".to_vec(),
15921 )
15922 .unwrap();
15923
15924 let error = handler.handle_control(conn, malformed).unwrap();
15925 assert_eq!(error[0].header.ty, FrameType::Error);
15926 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
15927
15928 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15929 let pong = handler.handle_control(conn, ping).unwrap();
15930 assert_eq!(pong[0].header.ty, FrameType::Pong);
15931 assert_eq!(pong[0].header.corr, 4);
15932 }
15933
15934 #[test]
15935 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
15936 let registry = Arc::new(Registry::default());
15937 let handler = ControlHandler::new(Arc::clone(®istry));
15938
15939 handler
15940 .handle_control(
15941 ConnectionId::new(1),
15942 hello_frame("aft", PROTOCOL_VERSION, 1),
15943 )
15944 .unwrap();
15945 let duplicate = handler
15946 .handle_control(
15947 ConnectionId::new(2),
15948 hello_frame("aft", PROTOCOL_VERSION, 2),
15949 )
15950 .unwrap();
15951
15952 assert_eq!(duplicate[0].header.ty, FrameType::Error);
15953 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15954 let registration = registry.get_module("aft").unwrap().unwrap();
15955 assert_eq!(registration.connection_id, ConnectionId::new(1));
15956 }
15957
15958 #[test]
15959 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
15960 let registry = Arc::new(Registry::default());
15961 let forwarding = Arc::new(ForwardingTable::default());
15962 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
15963 let handler =
15964 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15965 .with_process_liveness(process_liveness);
15966 let (ctx, route_channel, route_epoch) =
15967 bind_liveness_route(®istry, &forwarding, "aft-dead");
15968 let responses = handler
15969 .handle_route_poll(
15970 &ctx,
15971 route_poll_frame(41, PollKind::Liveness, route_channel),
15972 route_channel,
15973 route_epoch,
15974 PollKind::Liveness,
15975 )
15976 .unwrap();
15977
15978 assert_eq!(responses.len(), 1);
15979 assert_eq!(responses[0].header.ty, FrameType::Response);
15980 assert_route_poll_liveness(&responses[0], false);
15981 }
15982
15983 #[test]
15984 fn liveness_poll_without_process_source_uses_bound_route() {
15985 let registry = Arc::new(Registry::default());
15986 let forwarding = Arc::new(ForwardingTable::default());
15987 let handler =
15988 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15989 let (ctx, route_channel, route_epoch) =
15990 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
15991 let responses = handler
15992 .handle_route_poll(
15993 &ctx,
15994 route_poll_frame(42, PollKind::Liveness, route_channel),
15995 route_channel,
15996 route_epoch,
15997 PollKind::Liveness,
15998 )
15999 .unwrap();
16000
16001 assert_route_poll_liveness(&responses[0], true);
16002 }
16003
16004 #[test]
16005 fn liveness_poll_untracked_process_source_uses_bound_route() {
16006 let registry = Arc::new(Registry::default());
16007 let forwarding = Arc::new(ForwardingTable::default());
16008 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
16009 let handler =
16010 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16011 .with_process_liveness(process_liveness);
16012 let (ctx, route_channel, route_epoch) =
16013 bind_liveness_route(®istry, &forwarding, "aft-untracked");
16014 let responses = handler
16015 .handle_route_poll(
16016 &ctx,
16017 route_poll_frame(43, PollKind::Liveness, route_channel),
16018 route_channel,
16019 route_epoch,
16020 PollKind::Liveness,
16021 )
16022 .unwrap();
16023
16024 assert_route_poll_liveness(&responses[0], true);
16025 }
16026
16027 #[tokio::test]
16028 async fn unknown_op_returns_unknown_control_op() {
16029 let handler = ControlHandler::default();
16030 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
16031 let request = Frame::build(
16032 FrameType::Request,
16033 control_flags(),
16034 0,
16035 0,
16036 55,
16037 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
16038 )
16039 .unwrap();
16040
16041 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16042
16043 assert_eq!(response.len(), 1);
16044 assert_eq!(response[0].header.ty, FrameType::Error);
16045 assert_eq!(response[0].header.corr, 55);
16046 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
16047 }
16048
16049 #[tokio::test]
16050 async fn supervisor_provenance_rejects_unknown_exact_module() {
16051 let handler = ControlHandler::default();
16052 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
16053 let request = Frame::build(
16054 FrameType::Request,
16055 control_flags(),
16056 0,
16057 0,
16058 57,
16059 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
16060 )
16061 .unwrap();
16062
16063 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16064
16065 assert_eq!(response.len(), 1);
16066 assert_eq!(response[0].header.ty, FrameType::Error);
16067 assert_eq!(response[0].header.corr, 57);
16068 let error = parse_error(&response[0]);
16069 assert_eq!(error["code"], "unknown_module");
16070 assert_eq!(error["message"], "module_id 'missing' is not supervised");
16071 }
16072
16073 #[test]
16074 fn provenance_probe_override_keeps_handler_tests_deterministic() {
16075 let expected = subc_control::RunningImageAgreement::Unavailable {
16076 reason: subc_control::RunningImageUnavailableReason::HashFailed,
16077 };
16078 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
16079 assert_eq!(handler.provenance_probe_override, Some(expected));
16080 }
16081
16082 #[test]
16083 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
16084 let verdict = reload_verdict(
16085 std::path::Path::new("/bin/new"),
16086 Some(std::path::Path::new("/bin/old")),
16087 subc_control::RunningImageAgreement::Unavailable {
16088 reason: subc_control::RunningImageUnavailableReason::HashFailed,
16089 },
16090 );
16091 assert!(matches!(
16092 verdict.path,
16093 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
16094 if configured == std::path::Path::new("/bin/new")
16095 && spawned_from == std::path::Path::new("/bin/old")
16096 ));
16097 }
16098
16099 #[test]
16100 fn launch_spec_without_running_digest_is_unavailable_not_match() {
16101 use subc_control::{
16102 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
16103 };
16104 let configured = crate::supervise::ModuleSpec {
16105 carry_containment: false,
16106 module_id: "legacy".into(),
16107 program: "/module".into(),
16108 args: Vec::new(),
16109 env: Vec::new(),
16110 reserved: false,
16111 reserved_prefixes: Vec::new(),
16112 protocol: subc_control::ModuleProtocol::Subc,
16113 overlap: Default::default(),
16114 };
16115 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
16116 assert_eq!(
16117 launch_spec_agreement(&configured, false, Some(&digest)),
16118 Agreement::Unavailable {
16119 reason: Reason::NotRunning
16120 }
16121 );
16122 assert_eq!(
16123 launch_spec_agreement(&configured, true, None),
16124 Agreement::Unavailable {
16125 reason: Reason::SpawnedSpecUnavailable
16126 }
16127 );
16128 assert_eq!(
16129 launch_spec_agreement(&configured, true, Some(&digest)),
16130 Agreement::Match
16131 );
16132 }
16133
16134 #[cfg(windows)]
16135 #[test]
16136 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
16137 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
16138 let configured = dir.join("module.exe");
16139 std::fs::write(&configured, b"executable path fixture").unwrap();
16140 let spawned = std::fs::canonicalize(&configured).unwrap();
16141 assert_ne!(
16142 configured, spawned,
16143 "fixture must exercise different path spellings"
16144 );
16145 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
16146 let lower_case = configured.to_string_lossy().to_lowercase();
16147 for spelling in [
16148 &configured,
16149 Path::new(&forward_slashes),
16150 Path::new(&lower_case),
16151 ] {
16152 let verdict = reload_verdict(
16153 spelling,
16154 Some(&spawned),
16155 subc_control::RunningImageAgreement::Unavailable {
16156 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16157 },
16158 );
16159 assert_eq!(
16160 verdict.path,
16161 ReloadPathAgreement::Match,
16162 "configured {spelling:?}, spawned {spawned:?}"
16163 );
16164 }
16165 }
16166
16167 #[cfg(windows)]
16168 #[test]
16169 fn windows_reload_path_keeps_different_existing_program_pending() {
16170 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
16171 let old = dir.join("old.exe");
16172 let new = dir.join("new.exe");
16173 std::fs::write(&old, b"same executable content").unwrap();
16174 std::fs::copy(&old, &new).unwrap();
16175 let spawned = std::fs::canonicalize(&old).unwrap();
16176 let image = subc_control::RunningImageAgreement::Unavailable {
16177 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16178 };
16179 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
16180 assert!(
16181 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
16182 if configured == new && spawned_from == spawned)
16183 );
16184 let missing = dir.join("missing.exe");
16185 assert!(matches!(
16186 reload_verdict(&missing, Some(&spawned), image).path,
16187 ReloadPathAgreement::Mismatch { .. }
16188 ));
16189 }
16190
16191 #[test]
16192 fn reload_verdict_detects_replaced_image_at_same_path() {
16193 let image = subc_control::RunningImageAgreement::Mismatch {
16194 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
16195 digest: "old".into(),
16196 },
16197 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
16198 digest: "new".into(),
16199 },
16200 };
16201 let verdict = reload_verdict(
16202 std::path::Path::new("/bin/same"),
16203 Some(std::path::Path::new("/bin/same")),
16204 image.clone(),
16205 );
16206 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16207 assert_eq!(verdict.image, image);
16208 }
16209
16210 #[test]
16211 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
16212 let image = subc_control::RunningImageAgreement::Unavailable {
16213 reason: subc_control::RunningImageUnavailableReason::NotRunning,
16214 };
16215 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
16216 assert_eq!(
16217 verdict.path,
16218 subc_control::ReloadPathAgreement::Unavailable {
16219 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
16220 }
16221 );
16222 assert_eq!(verdict.image, image);
16223
16224 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
16225 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16226 };
16227 let verdict = reload_verdict(
16228 std::path::Path::new("/bin/same"),
16229 Some(std::path::Path::new("/bin/same")),
16230 unconfirmed.clone(),
16231 );
16232 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16233 assert_eq!(verdict.image, unconfirmed);
16234 }
16235
16236 #[test]
16237 fn reload_verdict_preserves_each_image_unavailability_reason() {
16238 use subc_control::RunningImageUnavailableReason as Reason;
16239
16240 for reason in [
16241 Reason::NotRunning,
16242 Reason::UnsupportedPlatform,
16243 Reason::RunningExecutableUnreadable,
16244 Reason::SpawnedPathUnreadable,
16245 Reason::HashFailed,
16246 Reason::ProcessIdentityUnconfirmed,
16247 Reason::Unknown("future_probe_reason".to_string()),
16248 ] {
16249 let image = subc_control::RunningImageAgreement::Unavailable {
16250 reason: reason.clone(),
16251 };
16252 let verdict = reload_verdict(
16253 std::path::Path::new("/bin/same"),
16254 Some(std::path::Path::new("/bin/same")),
16255 image.clone(),
16256 );
16257 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16258 assert_eq!(verdict.image, image, "{reason:?}");
16259 }
16260 }
16261
16262 #[tokio::test]
16263 async fn malformed_control_bodies_return_invalid_control_body() {
16264 let handler = ControlHandler::default();
16265 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
16266
16267 for (corr, body) in [
16268 (56, br#"{"route_channel":1}"#.as_slice()),
16269 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
16270 (
16271 58,
16272 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
16273 ),
16274 ] {
16275 let request = Frame::build(
16276 FrameType::Request,
16277 control_flags(),
16278 0,
16279 0,
16280 corr,
16281 body.to_vec(),
16282 )
16283 .unwrap();
16284 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16285
16286 assert_eq!(response.len(), 1);
16287 assert_eq!(response[0].header.ty, FrameType::Error);
16288 assert_eq!(response[0].header.corr, corr);
16289 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
16290 }
16291 }
16292
16293 #[tokio::test]
16294 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
16295 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16296 let registry = Arc::new(Registry::default());
16297 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
16298 let router = Router::with_control_handler(Arc::clone(&control));
16299 let connection = router.begin_connection();
16300 let (ctx, mut rx) = route_ctx(connection.id());
16301
16302 router
16303 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
16304 .await
16305 .unwrap();
16306 let response = rx.recv().await.unwrap();
16307 let ack = parse_ack(&response);
16308 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16309 let channel = 1;
16310
16311 let goodbye =
16312 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
16313 router.route_for_connection(&ctx, goodbye).await.unwrap();
16314 assert!(rx.try_recv().is_err());
16315 assert!(registry.get_module("aft").unwrap().is_none());
16316
16317 router
16318 .route_for_connection(&ctx, channel_request(channel, 13))
16319 .await
16320 .unwrap();
16321 let error_frame = rx.recv().await.unwrap();
16322 assert_eq!(error_frame.header.ty, FrameType::Error);
16323 assert_eq!(error_frame.header.channel, channel);
16324 let captured = crate::router::test_log::captured_logs(&logs);
16325 assert_eq!(
16326 captured
16327 .lines()
16328 .filter(|line| {
16329 line.contains("module_id=aft")
16330 && line.contains("reason=explicit_goodbye")
16331 && line.contains("module registration ended")
16332 })
16333 .count(),
16334 1,
16335 "unexpected GOODBYE registry log: {captured}"
16336 );
16337 }
16338
16339 #[tokio::test]
16340 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
16341 let registry = Arc::new(Registry::default());
16342 let handler = ControlHandler::new(registry).with_capability_config(
16343 [("prov".to_string(), true), ("cons".to_string(), true)],
16344 BTreeMap::new(),
16345 );
16346 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
16347 register_capability_manifest(
16348 &handler,
16349 &provider_ctx,
16350 &mut provider_rx,
16351 capability_manifest("prov", &["thing/v1"], &[]),
16352 1,
16353 )
16354 .await;
16355 let mut consumer = capability_manifest("cons", &[], &[]);
16356 consumer.capabilities.as_mut().unwrap().requires.push(
16357 subc_protocol::manifest::CapabilityRequirement {
16358 capability: "thing/v1".to_string(),
16359 need: subc_protocol::manifest::CapabilityNeed::Required,
16360 },
16361 );
16362 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
16363 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
16364 assert_eq!(
16365 handler.capability_evaluator.verdict("cons", "thing/v1"),
16366 Some(CapabilityVerdict::Provided)
16367 );
16368 let (mut client_rx, _) = open_route_for_capability_test(
16369 &handler,
16370 &provider_ctx,
16371 &mut provider_rx,
16372 703,
16373 3,
16374 "prov",
16375 None,
16376 )
16377 .await;
16378 let goodbye =
16379 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
16380 handler
16381 .handle_control_frame(&provider_ctx, goodbye)
16382 .await
16383 .unwrap();
16384 assert_eq!(
16385 handler.capability_evaluator.verdict("cons", "thing/v1"),
16386 Some(CapabilityVerdict::NeverProvided)
16387 );
16388 let closed = client_rx
16389 .try_recv()
16390 .expect("GOODBYE pushes route.closed before route GOODBYE");
16391 assert!(
16392 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
16393 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
16394 );
16395 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
16396 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
16397 }
16398
16399 struct RouteClosedFixture {
16400 handler: ControlHandler,
16401 #[cfg(unix)]
16402 supervisor: Supervisor,
16403 module_ctx: RouteCtx,
16404 #[cfg(unix)]
16405 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16406 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16407 route: Arc<crate::forwarding::RouteBinding>,
16408 }
16409
16410 async fn route_closed_fixture() -> RouteClosedFixture {
16411 let registry = Arc::new(Registry::default());
16412 let handler = ControlHandler::new(Arc::clone(®istry));
16413 #[cfg(unix)]
16414 let supervisor = Supervisor::new(registry, RestartPolicy::default())
16415 .with_forwarding(Arc::clone(&handler.forwarding));
16416 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
16417 register_capability_manifest(
16418 &handler,
16419 &module_ctx,
16420 &mut module_rx,
16421 capability_manifest("prov", &[], &[]),
16422 1,
16423 )
16424 .await;
16425 let (client_rx, bind) = open_route_for_capability_test(
16426 &handler,
16427 &module_ctx,
16428 &mut module_rx,
16429 702,
16430 2,
16431 "prov",
16432 None,
16433 )
16434 .await;
16435 let ModuleControlRequest::RouteBind {
16436 route_channel: channel,
16437 epoch: route_epoch,
16438 ..
16439 } = bind
16440 else {
16441 panic!("expected route.bind");
16442 };
16443 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
16444 handler
16445 .forwarding
16446 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
16447 .unwrap()
16448 else {
16449 panic!("the acknowledged route must be bound");
16450 };
16451 RouteClosedFixture {
16452 handler,
16453 #[cfg(unix)]
16454 supervisor,
16455 module_ctx,
16456 #[cfg(unix)]
16457 module_rx,
16458 client_rx,
16459 route,
16460 }
16461 }
16462
16463 impl RouteClosedFixture {
16464 #[cfg(unix)]
16465 async fn cut(&mut self) {
16466 let notices = async {
16467 let mut draining = self.module_rx.recv().await.unwrap();
16468 assert!(matches!(
16469 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
16470 &draining.body
16471 )
16472 .unwrap(),
16473 subc_protocol::session::ModuleControlCommand::Draining {
16474 reason: RouteCloseReason::Disable,
16475 ..
16476 }
16477 ));
16478 draining.flushed.take().unwrap().send(()).unwrap();
16479 let mut closing = self.client_rx.recv().await.unwrap();
16480 assert!(matches!(
16481 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
16482 ClientControlPush::RouteClosing {
16483 reason: RouteCloseReason::Restart,
16484 ..
16485 }
16486 ));
16487 closing.flushed.take().unwrap().send(()).unwrap();
16488 };
16489 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
16490 result.unwrap();
16491 }
16492
16493 fn disconnect(&mut self) -> ClientControlPush {
16494 self.handler
16495 .cleanup_connection(self.module_ctx.connection_id)
16496 .unwrap();
16497 let closed = self.client_rx.try_recv().unwrap();
16498 assert_eq!(closed.header.ty, FrameType::Push);
16499 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
16500 assert!(matches!(
16501 &push,
16502 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
16503 if channels == &[self.route.client_channel]
16504 ));
16505 assert_eq!(
16506 self.client_rx.try_recv().unwrap().header.ty,
16507 FrameType::Goodbye
16508 );
16509 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
16510 push
16511 }
16512 }
16513
16514 #[cfg(unix)]
16515 #[tokio::test(start_paused = true)]
16516 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
16517 let mut fixture = route_closed_fixture().await;
16518 fixture.cut().await;
16519 assert!(matches!(
16520 fixture.disconnect(),
16521 ClientControlPush::RouteClosed {
16522 reason: RouteCloseReason::Restart,
16523 drained: true,
16524 ..
16525 }
16526 ));
16527 }
16528
16529 #[cfg(unix)]
16530 #[tokio::test(start_paused = true)]
16531 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
16532 let mut fixture = route_closed_fixture().await;
16533 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
16534 let started = tokio::time::Instant::now();
16535 fixture.cut().await;
16536 assert!(started.elapsed() >= Duration::from_secs(2));
16537 assert_eq!(fixture.route.flow.in_flight(), 1);
16538 assert!(matches!(
16539 fixture.disconnect(),
16540 ClientControlPush::RouteClosed {
16541 reason: RouteCloseReason::Restart,
16542 drained: false,
16543 ..
16544 }
16545 ));
16546 }
16547
16548 #[tokio::test]
16549 async fn module_connection_crash_reports_crash_and_not_drained() {
16550 let mut fixture = route_closed_fixture().await;
16551 assert!(matches!(
16555 fixture.disconnect(),
16556 ClientControlPush::RouteClosed {
16557 reason: RouteCloseReason::Crash,
16558 drained: false,
16559 ..
16560 }
16561 ));
16562 }
16563
16564 #[tokio::test]
16565 async fn dropping_router_connection_releases_registration() {
16566 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16567 let registry = Arc::new(Registry::default());
16568 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
16569 let router = Router::with_control_handler(Arc::clone(&control));
16570 let connection = router.begin_connection();
16571 let connection_id = connection.id();
16572 let (ctx, mut rx) = route_ctx(connection_id);
16573
16574 router
16575 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
16576 .await
16577 .unwrap();
16578 let response = rx.recv().await.unwrap();
16579 let ack = parse_ack(&response);
16580 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16581 assert!(registry.get_module("aft").unwrap().is_some());
16582
16583 drop(connection);
16584
16585 assert!(registry.get_module("aft").unwrap().is_none());
16586 assert_eq!(registry.active_registration_count().unwrap(), 0);
16587
16588 control
16589 .cleanup_connection(ConnectionId::new(u64::MAX))
16590 .unwrap();
16591 let captured = crate::router::test_log::captured_logs(&logs);
16592 println!("captured registry lifecycle logs:\n{captured}");
16593 let events: Vec<_> = captured
16594 .lines()
16595 .filter(|line| {
16596 line.contains("module registered module_id=aft ")
16597 || line.contains("module registration ended")
16598 })
16599 .collect();
16600 assert_eq!(
16601 events.len(),
16602 2,
16603 "unexpected registry lifecycle logs: {captured}"
16604 );
16605 assert!(events[0].contains("module registered module_id=aft "));
16606 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
16607 assert!(events[1].contains(&format!(
16608 "module_id=aft connection_id={}",
16609 connection_id.get()
16610 )));
16611 assert!(events[1].contains("reason=connection_closed"));
16612 assert!(events[1].contains("module registration ended"));
16613 assert_eq!(
16614 captured
16615 .lines()
16616 .filter(|line| line.contains("module registered module_id=aft "))
16617 .count(),
16618 1,
16619 "legacy registration admission line must appear once: {captured}"
16620 );
16621 }
16622
16623 #[tokio::test]
16624 async fn hello_registration_keeps_legacy_module_registered_line_once() {
16625 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16626 let registry = Arc::new(Registry::default());
16627 let control = ControlHandler::new(Arc::clone(®istry));
16628 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
16629 hello_via_sink(
16630 &control,
16631 &ctx,
16632 &mut rx,
16633 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
16634 )
16635 .await;
16636
16637 let captured = crate::router::test_log::captured_logs(&logs);
16638 let admissions: Vec<_> = captured
16639 .lines()
16640 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
16641 .collect();
16642 assert_eq!(
16643 admissions.len(),
16644 1,
16645 "legacy admission line must remain exactly once: {captured}"
16646 );
16647 assert!(admissions[0].contains("routable_provider=true"));
16648 assert!(admissions[0].contains("connection_id=777"));
16649 }
16650
16651 fn capability_manifest(
16652 module_id: &str,
16653 provides: &[&str],
16654 must_never_reach: &[&str],
16655 ) -> ModuleManifest {
16656 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
16657 manifest.capabilities = Some(CapabilityDeclarations {
16658 provides: provides
16659 .iter()
16660 .map(|capability| (*capability).to_string())
16661 .collect(),
16662 requires: Vec::new(),
16663 must_never_reach: must_never_reach
16664 .iter()
16665 .map(|capability| (*capability).to_string())
16666 .collect(),
16667 });
16668 manifest
16669 }
16670
16671 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
16672 Frame::build(
16673 FrameType::Hello,
16674 control_flags(),
16675 0,
16676 0,
16677 corr,
16678 serde_json::to_vec(&ModuleHelloBody {
16679 protocol_ver: manifest.protocol_ver,
16680 manifest,
16681 control_ops: None,
16682 launch_nonce: None,
16683 })
16684 .expect("capability test HELLO serializes"),
16685 )
16686 .expect("capability test HELLO frame builds")
16687 }
16688
16689 fn catalog_update_with_capabilities_frame(
16690 corr: u64,
16691 capabilities: CapabilityDeclarations,
16692 ) -> Frame {
16693 Frame::build(
16694 FrameType::Request,
16695 control_flags(),
16696 0,
16697 0,
16698 corr,
16699 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16700 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
16701 capabilities: Some(capabilities),
16702 ready: None,
16703 })
16704 .expect("capability catalog.update serializes"),
16705 )
16706 .expect("capability catalog.update frame builds")
16707 }
16708
16709 async fn register_capability_manifest(
16710 handler: &ControlHandler,
16711 ctx: &RouteCtx,
16712 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16713 manifest: ModuleManifest,
16714 corr: u64,
16715 ) {
16716 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
16717 }
16718
16719 async fn open_route_for_capability_test(
16720 handler: &ControlHandler,
16721 target_ctx: &RouteCtx,
16722 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16723 client_connection_id: u64,
16724 corr: u64,
16725 target_module_id: &str,
16726 consumer_identity: Option<ConsumerIdentity>,
16727 ) -> (
16728 mpsc::Receiver<crate::router::OutboundFrame>,
16729 ModuleControlRequest,
16730 ) {
16731 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
16732 let route_handler = handler.clone();
16733 let target_module_id = target_module_id.to_string();
16734 let route_task = tokio::spawn(async move {
16735 route_handler
16736 .handle_control_frame(
16737 &client_ctx,
16738 route_open_frame_with_admission_facts(
16739 corr,
16740 &target_module_id,
16741 unique_project_root("admission-facts"),
16742 consumer_identity,
16743 None,
16744 ),
16745 )
16746 .await
16747 .expect("capability test route.open succeeds")
16748 });
16749 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
16750 .await
16751 .expect("capability test route.open must reach route.bind")
16752 .expect("target control receiver stays open");
16753 let bind_request: ModuleControlRequest =
16754 serde_json::from_slice(&bind.body).expect("route.bind decodes");
16755 handler
16756 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
16757 .await
16758 .expect("capability test route.bind ACK succeeds");
16759 assert!(route_task.await.expect("route.open task joins").is_empty());
16760 let opened = client_rx
16761 .recv()
16762 .await
16763 .expect("successful route.open publishes a response");
16764 assert!(matches!(
16765 serde_json::from_slice::<ClientControlResponse>(&opened.body),
16766 Ok(ClientControlResponse::RouteOpen { .. })
16767 ));
16768 (client_rx, bind_request)
16769 }
16770
16771 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
16772 assert_eq!(frame.header.ty, FrameType::Push);
16773 assert_eq!(frame.header.channel, 0);
16774 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
16775 .expect("route.closed control push decodes");
16776 let ClientControlPush::RouteClosed { channels, .. } = &push else {
16777 panic!("expected route.closed");
16778 };
16779 assert_eq!(channels.len(), 1, "exactly one violating route closed");
16780 let closed_channel = channels[0];
16781 let channels = channels.clone();
16782 assert_eq!(
16783 push,
16784 ClientControlPush::RouteClosed {
16785 module_id: target_module_id.to_string(),
16786 channels,
16787 reason: RouteCloseReason::CapabilityDenied,
16788 drained: false,
16789 abandoned: 0,
16790 excluded_subscriptions: 0,
16791 terminal: Some(false),
16792 }
16793 );
16794 closed_channel
16795 }
16796
16797 #[tokio::test]
16798 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
16799 let registry = Arc::new(Registry::default());
16800 let forwarding = Arc::new(ForwardingTable::default());
16801 let supervisor = SupervisorHandle::new();
16802 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16803 let handler =
16804 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16805 .with_supervisor(supervisor);
16806 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
16807 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
16808 register_capability_manifest(
16809 &handler,
16810 &target_ctx,
16811 &mut target_rx,
16812 capability_manifest("target", &["credentials-provider/v1"], &[]),
16813 1,
16814 )
16815 .await;
16816 register_capability_manifest(
16817 &handler,
16818 &opener_ctx,
16819 &mut opener_rx,
16820 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16821 2,
16822 )
16823 .await;
16824
16825 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
16826 let replies = handler
16827 .handle_control_frame(
16828 &client_ctx,
16829 route_open_frame_with_admission_facts(
16830 3,
16831 "target",
16832 unique_project_root("admission-facts"),
16833 Some(ConsumerIdentity {
16834 module_id: "opener".to_string(),
16835 launch_nonce: "opener-nonce".to_string(),
16836 }),
16837 None,
16838 ),
16839 )
16840 .await
16841 .expect("denied route.open returns a typed frame");
16842 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16843 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16844 assert!(
16845 target_rx.try_recv().is_err(),
16846 "forbidden route.open must not relay route.bind"
16847 );
16848 }
16849
16850 #[tokio::test]
16851 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
16852 let registry = Arc::new(Registry::default());
16853 let forwarding = Arc::new(ForwardingTable::default());
16854 let supervisor = SupervisorHandle::new();
16855 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16856 let handler =
16857 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16858 .with_supervisor(supervisor);
16859 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
16860 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
16861 register_capability_manifest(
16862 &handler,
16863 &target_ctx,
16864 &mut target_rx,
16865 capability_manifest("target", &["credentials-provider/v1"], &[]),
16866 1,
16867 )
16868 .await;
16869 register_capability_manifest(
16870 &handler,
16871 &old_opener_ctx,
16872 &mut old_opener_rx,
16873 capability_manifest("opener", &[], &[]),
16874 2,
16875 )
16876 .await;
16877 let (mut client_rx, _) = open_route_for_capability_test(
16878 &handler,
16879 &target_ctx,
16880 &mut target_rx,
16881 712,
16882 3,
16883 "target",
16884 Some(ConsumerIdentity {
16885 module_id: "opener".to_string(),
16886 launch_nonce: "opener-nonce".to_string(),
16887 }),
16888 )
16889 .await;
16890 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16891
16892 handler
16893 .cleanup_connection(old_opener_ctx.connection_id)
16894 .expect("old opener registration cleans up");
16895 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
16896 register_capability_manifest(
16897 &handler,
16898 &new_opener_ctx,
16899 &mut new_opener_rx,
16900 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16901 4,
16902 )
16903 .await;
16904
16905 let closed_channel = assert_capability_denied_push(
16906 client_rx
16907 .try_recv()
16908 .expect("HELLO deny addition must emit route.closed")
16909 .frame,
16910 "target",
16911 );
16912 let goodbye = client_rx
16913 .try_recv()
16914 .expect("HELLO deny addition must send the client a route GOODBYE")
16915 .frame;
16916 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16917 assert_eq!(goodbye.header.channel, closed_channel);
16918 assert_ne!(goodbye.header.channel, 0);
16919 assert_ne!(goodbye.header.epoch, 0);
16920 assert!(client_rx.try_recv().is_err());
16921 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16922 assert!(matches!(
16923 target_rx.try_recv(),
16924 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16925 ));
16926 }
16927
16928 #[tokio::test]
16929 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
16930 let registry = Arc::new(Registry::default());
16931 let forwarding = Arc::new(ForwardingTable::default());
16932 let supervisor = SupervisorHandle::new();
16933 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16934 let handler =
16935 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16936 .with_supervisor(supervisor);
16937 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
16938 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
16939 register_capability_manifest(
16940 &handler,
16941 &target_ctx,
16942 &mut target_rx,
16943 capability_manifest("target", &[], &[]),
16944 1,
16945 )
16946 .await;
16947 register_capability_manifest(
16948 &handler,
16949 &opener_ctx,
16950 &mut opener_rx,
16951 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16952 2,
16953 )
16954 .await;
16955 let (mut client_rx, _) = open_route_for_capability_test(
16956 &handler,
16957 &target_ctx,
16958 &mut target_rx,
16959 722,
16960 3,
16961 "target",
16962 Some(ConsumerIdentity {
16963 module_id: "opener".to_string(),
16964 launch_nonce: "opener-nonce".to_string(),
16965 }),
16966 )
16967 .await;
16968 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16969
16970 let replies = handler
16971 .handle_control_frame(
16972 &target_ctx,
16973 catalog_update_with_capabilities_frame(
16974 4,
16975 CapabilityDeclarations {
16976 provides: vec!["credentials-provider/v1".to_string()],
16977 requires: Vec::new(),
16978 must_never_reach: Vec::new(),
16979 },
16980 ),
16981 )
16982 .await
16983 .expect("claim catalog.update succeeds");
16984 assert!(matches!(
16985 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
16986 Ok(ModuleControlResponseToModule::CatalogUpdate {})
16987 ));
16988 let closed_channel = assert_capability_denied_push(
16989 client_rx
16990 .try_recv()
16991 .expect("claim addition must emit route.closed")
16992 .frame,
16993 "target",
16994 );
16995 let goodbye = client_rx
16996 .try_recv()
16997 .expect("claim addition must send the client a route GOODBYE")
16998 .frame;
16999 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17000 assert_eq!(goodbye.header.channel, closed_channel);
17001 assert_ne!(goodbye.header.channel, 0);
17002 assert_ne!(goodbye.header.epoch, 0);
17003 assert!(client_rx.try_recv().is_err());
17004 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17005 assert!(matches!(
17006 target_rx.try_recv(),
17007 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
17008 ));
17009 }
17010
17011 #[tokio::test]
17012 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
17013 let registry = Arc::new(Registry::default());
17014 let forwarding = Arc::new(ForwardingTable::default());
17015 let supervisor = SupervisorHandle::new();
17016 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17017 let handler =
17018 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17019 .with_supervisor(supervisor);
17020 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
17021 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
17022 register_capability_manifest(
17023 &handler,
17024 &target_ctx,
17025 &mut target_rx,
17026 capability_manifest("target", &["credentials-provider/v1"], &[]),
17027 1,
17028 )
17029 .await;
17030 register_capability_manifest(
17031 &handler,
17032 &opener_ctx,
17033 &mut opener_rx,
17034 capability_manifest("opener", &[], &[]),
17035 2,
17036 )
17037 .await;
17038 let (mut client_rx, _) = open_route_for_capability_test(
17039 &handler,
17040 &target_ctx,
17041 &mut target_rx,
17042 732,
17043 3,
17044 "target",
17045 Some(ConsumerIdentity {
17046 module_id: "opener".to_string(),
17047 launch_nonce: "opener-nonce".to_string(),
17048 }),
17049 )
17050 .await;
17051 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17052
17053 handler
17054 .handle_control_frame(
17055 &target_ctx,
17056 catalog_update_with_capabilities_frame(
17057 4,
17058 CapabilityDeclarations {
17059 provides: Vec::new(),
17060 requires: Vec::new(),
17061 must_never_reach: Vec::new(),
17062 },
17063 ),
17064 )
17065 .await
17066 .expect("claim removal catalog.update succeeds");
17067 assert_eq!(
17068 forwarding.active_binding_count().unwrap(),
17069 1,
17070 "removing an attested target claim must leave the route census unchanged"
17071 );
17072 assert!(
17073 client_rx.try_recv().is_err(),
17074 "claim removal must not emit route.closed or a client route GOODBYE"
17075 );
17076 assert!(
17077 target_rx.try_recv().is_err(),
17078 "claim removal must not send the target a route GOODBYE"
17079 );
17080 }
17081
17082 #[tokio::test]
17085 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
17086 let registry = Arc::new(Registry::default());
17087 let forwarding = Arc::new(ForwardingTable::default());
17088 let supervisor = SupervisorHandle::new();
17089 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
17090 let handler =
17091 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17092 .with_supervisor(supervisor);
17093 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
17094 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
17095 register_capability_manifest(
17096 &handler,
17097 &target_ctx,
17098 &mut target_rx,
17099 capability_manifest("target", &["credentials-provider/v1"], &[]),
17100 1,
17101 )
17102 .await;
17103 register_capability_manifest(
17104 &handler,
17105 &opener_ctx,
17106 &mut opener_rx,
17107 capability_manifest("opener", &[], &["credentials-provider/v1"]),
17108 2,
17109 )
17110 .await;
17111
17112 let (_client_rx, bind) = open_route_for_capability_test(
17113 &handler,
17114 &target_ctx,
17115 &mut target_rx,
17116 742,
17117 3,
17118 "target",
17119 None,
17120 )
17121 .await;
17122 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
17123 panic!("direct scope-honesty route must bind");
17124 };
17125 assert_eq!(principal, Some(Principal::Direct));
17126 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17127 }
17128
17129 #[tokio::test]
17132 async fn must_never_reach_self_route_is_capability_forbidden() {
17133 let registry = Arc::new(Registry::default());
17134 let forwarding = Arc::new(ForwardingTable::default());
17135 let supervisor = SupervisorHandle::new();
17136 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
17137 let handler =
17138 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17139 .with_supervisor(supervisor);
17140 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
17141 register_capability_manifest(
17142 &handler,
17143 &self_ctx,
17144 &mut self_rx,
17145 capability_manifest(
17146 "self-provider",
17147 &["credentials-provider/v1"],
17148 &["credentials-provider/v1"],
17149 ),
17150 1,
17151 )
17152 .await;
17153
17154 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
17155 let replies = handler
17156 .handle_control_frame(
17157 &client_ctx,
17158 route_open_frame_with_admission_facts(
17159 2,
17160 "self-provider",
17161 unique_project_root("admission-facts"),
17162 Some(ConsumerIdentity {
17163 module_id: "self-provider".to_string(),
17164 launch_nonce: "self-nonce".to_string(),
17165 }),
17166 None,
17167 ),
17168 )
17169 .await
17170 .expect("self-route refusal returns a typed frame");
17171 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
17172 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17173 assert!(
17174 self_rx.try_recv().is_err(),
17175 "self denial must not relay route.bind"
17176 );
17177 }
17178
17179 #[test]
17180 fn unsupported_channel_zero_frame_returns_error() {
17181 let handler = ControlHandler::default();
17182 let request = Frame::build(
17183 FrameType::Request,
17184 control_flags(),
17185 0,
17186 0,
17187 21,
17188 b"opaque".to_vec(),
17189 )
17190 .unwrap();
17191
17192 let response = handler
17193 .handle_control(ConnectionId::new(1), request)
17194 .unwrap();
17195
17196 assert_eq!(response[0].header.ty, FrameType::Error);
17197 assert_eq!(
17198 parse_error(&response[0])["code"],
17199 "unsupported_control_frame"
17200 );
17201 }
17202
17203 mod swap {
17208 use super::*;
17209
17210 const INCUMBENT: ConnectionId = ConnectionId::new(30);
17211 const CANDIDATE: ConnectionId = ConnectionId::new(40);
17212
17213 struct Swap {
17214 registry: Arc<Registry>,
17215 forwarding: Arc<ForwardingTable>,
17216 handler: ControlHandler,
17217 incumbent_ctx: RouteCtx,
17218 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17219 candidate_ctx: RouteCtx,
17220 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17221 }
17222
17223 async fn swap_with_incumbent() -> Swap {
17224 let registry = Arc::new(Registry::default());
17225 let forwarding = Arc::new(ForwardingTable::default());
17226 let handler =
17227 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17228 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
17229 hello_via_sink(
17230 &handler,
17231 &incumbent_ctx,
17232 &mut incumbent_rx,
17233 hello_frame("aft", PROTOCOL_VERSION, 7),
17234 )
17235 .await;
17236 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
17237 Swap {
17238 registry,
17239 forwarding,
17240 handler,
17241 incumbent_ctx,
17242 incumbent_rx,
17243 candidate_ctx,
17244 candidate_rx,
17245 }
17246 }
17247
17248 fn register_candidate(swap: &Swap, ready: Option<bool>) {
17249 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
17250 candidate_manifest.ready = ready;
17251 let registration = swap
17252 .registry
17253 .register_candidate_with_control_ops(
17254 candidate_manifest,
17255 PROTOCOL_VERSION,
17256 CANDIDATE,
17257 module_baseline_control_ops(),
17258 )
17259 .unwrap();
17260 swap.forwarding
17261 .register_candidate_module_connection(
17262 CANDIDATE,
17263 "aft".to_string(),
17264 PROTOCOL_VERSION,
17265 manifest_concurrency(®istration.manifest),
17266 swap.candidate_ctx.egress.clone(),
17267 )
17268 .unwrap();
17269 }
17270
17271 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
17272 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
17273 swap.registry.promote_candidate("aft").unwrap().unwrap();
17274 cutover.incumbent.unwrap()
17275 }
17276
17277 #[tokio::test]
17286 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
17287 let mut swap = swap_with_incumbent().await;
17288 let hold = Arc::new(std::sync::Barrier::new(2));
17289 let handler = swap
17290 .handler
17291 .clone()
17292 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
17293 handler
17294 .supervisor
17295 .open_swap("aft", "candidate-nonce".to_string());
17296 let hello = {
17297 let handler = handler.clone();
17298 let candidate_ctx = swap.candidate_ctx.clone();
17299 std::thread::spawn(move || {
17300 handler
17301 .handle_hello(
17302 candidate_ctx.connection_id,
17303 Some(candidate_ctx.egress.clone()),
17304 hello_frame_with_nonce(
17305 "aft",
17306 PROTOCOL_VERSION,
17307 8,
17308 Some("candidate-nonce"),
17309 ),
17310 )
17311 .unwrap()
17312 })
17313 };
17314
17315 hold.wait();
17316 assert!(swap
17319 .registry
17320 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
17321 .unwrap()
17322 .is_some());
17323 assert!(swap
17324 .registry
17325 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
17326 .unwrap()
17327 .is_none());
17328 assert!(swap
17329 .forwarding
17330 .module_endpoint_for_connection(CANDIDATE)
17331 .unwrap()
17332 .is_none());
17333
17334 hold.wait();
17335 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
17336 assert_eq!(
17337 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
17338 FrameType::HelloAck
17339 );
17340 let published = swap
17341 .registry
17342 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
17343 .unwrap()
17344 .expect("candidate is published once its HELLO completes");
17345 assert_eq!(published.connection_id, CANDIDATE);
17346 assert!(swap
17347 .forwarding
17348 .module_endpoint_for_connection(CANDIDATE)
17349 .unwrap()
17350 .is_some());
17351 }
17352
17353 fn keyed_total(counters: &Value, key: &str) -> u64 {
17354 counters[key]
17355 .as_object()
17356 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
17357 .unwrap_or(0)
17358 }
17359
17360 #[tokio::test]
17361 async fn replacement_logs_old_end_and_new_admission_once() {
17362 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
17363 let mut swap = swap_with_incumbent().await;
17364 swap.handler
17365 .supervisor
17366 .open_swap("aft", "candidate-nonce".to_string());
17367 hello_via_sink(
17368 &swap.handler,
17369 &swap.candidate_ctx,
17370 &mut swap.candidate_rx,
17371 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
17372 )
17373 .await;
17374 cutover(&swap);
17375 swap.handler.cleanup_connection(INCUMBENT).unwrap();
17376
17377 let captured = crate::router::test_log::captured_logs(&logs);
17378 println!("captured replacement registry logs:\n{captured}");
17379 let new_admissions: Vec<_> = captured
17380 .lines()
17381 .filter(|line| {
17382 line.contains("connection_id=40")
17383 && line.contains("swap candidate registered; not routable until cutover")
17384 })
17385 .collect();
17386 assert_eq!(
17387 new_admissions.len(),
17388 1,
17389 "unexpected admission logs: {captured}"
17390 );
17391 assert!(new_admissions[0].contains("module_id=aft"));
17392 assert!(new_admissions[0].contains("connection_id=40"));
17393 assert!(
17394 new_admissions[0].contains("swap candidate registered; not routable until cutover")
17395 );
17396 assert!(new_admissions[0].contains("ready=true"));
17397
17398 let old_admissions: Vec<_> = captured
17399 .lines()
17400 .filter(|line| {
17401 line.contains("module registered module_id=aft ")
17402 && line.contains("connection_id=30")
17403 })
17404 .collect();
17405 assert_eq!(
17406 old_admissions.len(),
17407 1,
17408 "unexpected incumbent admission: {captured}"
17409 );
17410
17411 let old_ends: Vec<_> = captured
17412 .lines()
17413 .filter(|line| {
17414 line.contains("connection_id=30") && line.contains("module registration ended")
17415 })
17416 .collect();
17417 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
17418 assert!(old_ends[0].contains("module_id=aft"));
17419 assert!(old_ends[0].contains("connection_id=30"));
17420 assert!(old_ends[0].contains("reason=replaced"));
17421 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
17422 assert_eq!(
17423 captured
17424 .lines()
17425 .filter(|line| line.contains("module registration promoted"))
17426 .count(),
17427 1,
17428 "unexpected promotion logs: {captured}"
17429 );
17430 }
17431
17432 #[tokio::test]
17438 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
17439 let mut swap = swap_with_incumbent().await;
17440 let handler = swap.handler.clone();
17441
17442 let cotenant = ConnectionId::new(31);
17444 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
17445 let (cotenant_task, cotenant_bind) = relay_route_open(
17446 &handler,
17447 cotenant,
17448 &cotenant_ctx.egress,
17449 &mut swap.incumbent_rx,
17450 100,
17451 "aft",
17452 "swap-cotenant",
17453 )
17454 .await;
17455 handler
17456 .handle_control_frame(
17457 &swap.incumbent_ctx,
17458 route_bind_ack(cotenant_bind.header.corr),
17459 )
17460 .await
17461 .unwrap();
17462 assert!(cotenant_task.await.unwrap().is_empty());
17463 let (cotenant_channel, cotenant_epoch) =
17464 published_route(&cotenant_rx.recv().await.unwrap());
17465
17466 let caller = ConnectionId::new(32);
17468 let (caller_ctx, mut caller_rx) = route_ctx(caller);
17469 let (caller_task, caller_bind) = relay_route_open(
17470 &handler,
17471 caller,
17472 &caller_ctx.egress,
17473 &mut swap.incumbent_rx,
17474 101,
17475 "aft",
17476 "swap-caller",
17477 )
17478 .await;
17479 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
17480
17481 register_candidate(&swap, None);
17482 cutover(&swap);
17483
17484 let ack = handler
17486 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
17487 .await;
17488 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
17489 if module_loop_error.is_some() {
17490 handler.cleanup_connection(INCUMBENT).unwrap();
17493 }
17494
17495 assert!(
17497 cotenant_rx.try_recv().is_err(),
17498 "the co-tenant route on the incumbent was torn down by one late ack: \
17499 {module_loop_error:?}"
17500 );
17501 assert!(matches!(
17502 swap.forwarding
17503 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
17504 .unwrap(),
17505 DataRoute::Client(DataRouteState::Bound(_))
17506 ));
17507 assert_eq!(module_loop_error, None);
17508 assert!(swap
17509 .registry
17510 .get_module_by_connection(INCUMBENT)
17511 .unwrap()
17512 .is_some());
17513
17514 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
17516 .await
17517 .expect("the incumbent is told to drop the abandoned binding")
17518 .unwrap()
17519 .frame;
17520 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17521 assert_eq!(goodbye.header.channel, abandoned_channel);
17522 assert_eq!(goodbye.header.epoch, abandoned_epoch);
17523 assert!(swap.incumbent_rx.try_recv().is_err());
17524
17525 let response = caller_task.await.unwrap();
17527 assert_eq!(response.len(), 1);
17528 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17529 assert!(caller_rx.try_recv().is_err());
17530
17531 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
17535 let counters = handler.counters().snapshot();
17536 assert_eq!(
17537 keyed_total(&counters, "route_open_accepted_by_principal"),
17538 1
17539 );
17540 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
17541 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
17542 }
17543
17544 #[tokio::test]
17548 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
17549 let mut swap = swap_with_incumbent().await;
17550 register_candidate(&swap, None);
17551 let incumbent = cutover(&swap);
17552 swap.forwarding
17553 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
17554 .unwrap()
17555 .expect("the incumbent is still registered");
17556
17557 let client = ConnectionId::new(33);
17558 let (client_ctx, mut client_rx) = route_ctx(client);
17559 let route_handler = swap.handler.clone();
17560 let open_ctx = RouteCtx {
17561 connection_id: client,
17562 egress: client_ctx.egress.clone(),
17563 };
17564 let mut route_task = tokio::spawn(async move {
17565 route_handler
17566 .handle_control_frame(
17567 &open_ctx,
17568 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
17569 )
17570 .await
17571 .unwrap()
17572 });
17573 let bind = tokio::select! {
17574 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
17575 response = &mut route_task => {
17576 let response = response.unwrap();
17577 panic!(
17578 "post-cutover route.open was refused instead of relayed to the candidate: {}",
17579 parse_error(&response[0])["code"]
17580 );
17581 }
17582 };
17583 swap.handler
17584 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
17585 .await
17586 .unwrap();
17587 assert!(route_task.await.unwrap().is_empty());
17588 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
17589 match swap
17590 .forwarding
17591 .lookup_data_route(client, channel, epoch)
17592 .unwrap()
17593 {
17594 DataRoute::Client(DataRouteState::Bound(route)) => {
17595 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
17596 }
17597 other => panic!("expected a bound route on the candidate, got {other:?}"),
17598 }
17599 assert!(swap.incumbent_rx.try_recv().is_err());
17600 }
17601
17602 #[tokio::test]
17607 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
17608 let swap = swap_with_incumbent().await;
17609 register_candidate(&swap, Some(false));
17610 let update = Frame::build(
17611 FrameType::Request,
17612 control_flags(),
17613 0,
17614 0,
17615 55,
17616 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
17617 provides: manifest("aft", PROTOCOL_VERSION).provides,
17618 capabilities: None,
17619 ready: Some(true),
17620 })
17621 .unwrap(),
17622 )
17623 .unwrap();
17624
17625 let replies = swap
17626 .handler
17627 .handle_control_frame(&swap.candidate_ctx, update)
17628 .await
17629 .unwrap();
17630
17631 assert_eq!(replies.len(), 1);
17632 assert_eq!(
17633 replies[0].header.ty,
17634 FrameType::Response,
17635 "candidate catalog.update was refused: {:?}",
17636 serde_json::from_slice::<Value>(&replies[0].body).ok()
17637 );
17638 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
17639 assert_eq!(
17640 swap.registry
17641 .get_module("aft")
17642 .unwrap()
17643 .unwrap()
17644 .connection_id,
17645 INCUMBENT
17646 );
17647 }
17648 }
17649
17650 mod swap_admission {
17654 use super::*;
17655
17656 const INCUMBENT_NONCE: &str = "incumbent-nonce";
17657 const CANDIDATE_NONCE: &str = "candidate-nonce";
17658
17659 fn handler_with_incumbent(
17660 module_id: &str,
17661 reserved: bool,
17662 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
17663 let registry = Arc::new(Registry::default());
17664 let supervisor = SupervisorHandle::new();
17665 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
17666 if reserved {
17667 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
17668 }
17669 let handler =
17670 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
17671 let incumbent = handler
17672 .handle_control(
17673 ConnectionId::new(1),
17674 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
17675 )
17676 .unwrap();
17677 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
17678 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
17679 (registry, supervisor, handler)
17680 }
17681
17682 #[test]
17688 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
17689 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
17690
17691 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
17692 let replies = handler
17693 .handle_control(
17694 ConnectionId::new(connection),
17695 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
17696 )
17697 .unwrap();
17698 assert_eq!(replies[0].header.ty, FrameType::Error);
17699 assert_eq!(
17700 parse_error(&replies[0])["code"],
17701 "swap_token_invalid",
17702 "nonce {nonce:?}"
17703 );
17704 }
17705 assert!(registry.get_candidate("aft").unwrap().is_none());
17706 assert_eq!(
17707 registry.get_module("aft").unwrap().unwrap().connection_id,
17708 ConnectionId::new(1)
17709 );
17710
17711 let admitted = handler
17714 .handle_control(
17715 ConnectionId::new(4),
17716 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
17717 )
17718 .unwrap();
17719 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
17720 assert_eq!(
17721 registry
17722 .get_candidate("aft")
17723 .unwrap()
17724 .unwrap()
17725 .connection_id,
17726 ConnectionId::new(4)
17727 );
17728 assert_eq!(
17729 registry.get_module("aft").unwrap().unwrap().connection_id,
17730 ConnectionId::new(1),
17731 "the candidate must not take the active slot"
17732 );
17733 let replayed = handler
17734 .handle_control(
17735 ConnectionId::new(5),
17736 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
17737 )
17738 .unwrap();
17739 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
17740
17741 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
17745 let squatter = handler
17746 .handle_control(
17747 ConnectionId::new(6),
17748 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
17749 )
17750 .unwrap();
17751 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
17752 assert!(
17753 registry.get_module("aft").unwrap().is_none(),
17754 "a squatter took the active slot of an id being swapped"
17755 );
17756 }
17757
17758 #[test]
17763 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
17764 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
17765
17766 let replies = handler
17767 .handle_control(
17768 ConnectionId::new(2),
17769 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17770 )
17771 .unwrap();
17772
17773 assert_eq!(
17774 replies[0].header.ty,
17775 FrameType::HelloAck,
17776 "reserved candidate refused: {:?}",
17777 serde_json::from_slice::<Value>(&replies[0].body).ok()
17778 );
17779 assert_eq!(
17780 registry
17781 .get_candidate("vault")
17782 .unwrap()
17783 .unwrap()
17784 .connection_id,
17785 ConnectionId::new(2)
17786 );
17787 }
17788
17789 #[test]
17792 fn without_an_open_swap_the_ordinary_gates_decide() {
17793 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
17794 supervisor.close_swap("vault");
17795
17796 let candidate = handler
17797 .handle_control(
17798 ConnectionId::new(2),
17799 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17800 )
17801 .unwrap();
17802 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
17803 let duplicate = handler
17804 .handle_control(
17805 ConnectionId::new(3),
17806 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
17807 )
17808 .unwrap();
17809 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
17810 assert!(registry.get_candidate("vault").unwrap().is_none());
17811 }
17812 }
17813
17814 mod scopes {
17818 mod idle_tests {
17819 use super::*;
17820 include!("scope_idle_tests.rs");
17821 }
17822 mod route_open_timings {
17823 use super::*;
17824 include!("route_open_timing_tests.rs");
17825 }
17826 mod route_open_refusal_log {
17827 use super::*;
17828 include!("route_open_refusal_log_tests.rs");
17829 }
17830 mod activity {
17831 include!("activity_scope_tests.rs");
17832 }
17833 mod delta {
17834 include!("scope_handler_delta_tests.rs");
17835 }
17836 mod call_modes {
17837 include!("scope_call_mode_tests.rs");
17838 }
17839 mod federated {
17840 include!("scope_federated_tests.rs");
17841 }
17842 #[tokio::test]
17843 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
17844 let rig = rig().await;
17845 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
17846 "s".into(),
17847 1,
17848 "tool".into(),
17849 ))
17850 .unwrap();
17851 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
17852 let (direct, _rx) = wide_ctx(999);
17853 for (ctx, expected) in [
17854 (&rig.owner, FrameType::Response),
17855 (&direct, FrameType::Error),
17856 ] {
17857 let frame =
17858 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
17859 .unwrap();
17860 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
17861 assert_eq!(replies[0].header.ty, expected);
17862 if expected == FrameType::Response {
17863 let reply: subc_control::ScopeServedByReply =
17864 serde_json::from_slice(&replies[0].body).unwrap();
17865 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
17866 } else {
17867 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
17868 }
17869 }
17870 let reply = call(
17871 &rig.handler,
17872 &rig.owner,
17873 &ModuleControlRequestFromModule::ScopeDescribe {
17874 owner: Principal::Reserved {
17875 module_id: OWNER.into(),
17876 },
17877 scope_ref: "s".into(),
17878 },
17879 )
17880 .await;
17881 assert!(matches!(
17882 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
17883 ModuleControlResponseToModule::ScopeDescribe { .. }
17884 ));
17885 }
17886
17887 use subc_protocol::scope::{
17888 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
17889 ScopeStatus,
17890 };
17891
17892 use super::*;
17893
17894 const OWNER: &str = "prefrontal-core";
17895
17896 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
17897 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
17898 }
17899
17900 async fn call(
17901 handler: &ControlHandler,
17902 ctx: &RouteCtx,
17903 request: &ModuleControlRequestFromModule,
17904 ) -> Frame {
17905 let body = serde_json::to_vec(request).unwrap();
17906 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
17907 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
17908 assert_eq!(replies.len(), 1, "{replies:?}");
17909 replies.pop().unwrap()
17910 }
17911
17912 async fn sync(
17913 handler: &ControlHandler,
17914 ctx: &RouteCtx,
17915 generation: u64,
17916 scopes: Vec<ScopeRecord>,
17917 ) -> Result<ModuleControlResponseToModule, String> {
17918 let reply = call(
17919 handler,
17920 ctx,
17921 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
17922 )
17923 .await;
17924 match reply.header.ty {
17925 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
17926 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
17927 }
17928 }
17929
17930 async fn describe(
17931 handler: &ControlHandler,
17932 ctx: &RouteCtx,
17933 owner: &str,
17934 scope_ref: &str,
17935 ) -> ModuleControlResponseToModule {
17936 let reply = call(
17937 handler,
17938 ctx,
17939 &ModuleControlRequestFromModule::ScopeDescribe {
17940 owner: Principal::Reserved {
17941 module_id: owner.to_string(),
17942 },
17943 scope_ref: scope_ref.to_string(),
17944 },
17945 )
17946 .await;
17947 assert_eq!(
17948 reply.header.ty,
17949 FrameType::Response,
17950 "{:?}",
17951 parse_error(&reply)
17952 );
17953 serde_json::from_slice(&reply.body).unwrap()
17954 }
17955
17956 #[tokio::test]
17957 async fn scope_describe_never_echoes_bind_proof_or_hash() {
17958 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17959 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17960 let mut rig = rig().await;
17961 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17962 .await;
17963 let _bound = rig
17964 .bound(
17965 Some(AFT),
17966 PLEXUS,
17967 Some(
17968 rig.selector("s", Some(1))
17969 .with_bind_proof(Some(PROOF.into())),
17970 ),
17971 )
17972 .await;
17973 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
17974 let output = serde_json::to_string(&reply).unwrap();
17975 assert!(output.contains("\"status\":\"live\""));
17976 assert!(!output.contains(PROOF));
17977 assert!(!output.contains(HASH));
17978 }
17979
17980 async fn module(
17982 handler: &ControlHandler,
17983 connection: u64,
17984 module_id: &str,
17985 nonce: Option<&str>,
17986 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17987 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
17988 hello_via_sink(
17989 handler,
17990 &ctx,
17991 &mut rx,
17992 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
17993 )
17994 .await;
17995 (ctx, rx)
17996 }
17997
17998 #[tokio::test]
18001 async fn a_client_connection_cannot_sync_or_describe() {
18002 let handler = ControlHandler::new(Arc::new(Registry::default()));
18003 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
18004 for request in [
18005 ModuleControlRequestFromModule::ScopeSync {
18006 generation: 1,
18007 scopes: vec![head("s", 1)],
18008 },
18009 ModuleControlRequestFromModule::ScopeDescribe {
18010 owner: Principal::Direct,
18011 scope_ref: "s".to_string(),
18012 },
18013 ] {
18014 let reply = call(&handler, &ctx, &request).await;
18015 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
18016 }
18017 assert!(
18018 !handler
18019 .scopes
18020 .read()
18021 .unwrap()
18022 .describe(
18023 &Principal::Reserved {
18024 module_id: OWNER.to_string()
18025 },
18026 "s"
18027 )
18028 .owner_synced
18029 );
18030 }
18031
18032 #[tokio::test]
18035 async fn a_module_without_a_supervised_launch_cannot_sync() {
18036 let handler = ControlHandler::new(Arc::new(Registry::default()));
18037 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
18038 assert_eq!(
18039 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
18040 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18041 );
18042 }
18043
18044 #[tokio::test]
18045 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
18046 let supervisor = SupervisorHandle::new();
18047 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
18048 let handler = ControlHandler::new(Arc::new(Registry::default()))
18049 .with_supervisor(supervisor.clone());
18050 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
18051 sync(&handler, &incumbent, 1, vec![head("s", 1)])
18052 .await
18053 .expect("the current launch syncs");
18054
18055 supervisor.open_swap(OWNER, "n2".to_string());
18058 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
18059 assert_eq!(
18060 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
18061 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18062 );
18063 sync(&handler, &incumbent, 2, vec![head("s", 1)])
18064 .await
18065 .expect("the serving owner syncs during the swap");
18066
18067 supervisor.close_swap(OWNER);
18070 assert_eq!(
18071 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
18072 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18073 );
18074 sync(&handler, &incumbent, 3, vec![head("s", 1)])
18075 .await
18076 .expect("the serving owner syncs after the rollback");
18077 handler.cleanup_connection(candidate.connection_id).unwrap();
18078
18079 supervisor.open_swap(OWNER, "n3".to_string());
18084 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
18085 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
18086 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
18087 .await
18088 .expect("the promoted launch takes authority");
18089 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
18090 panic!("unexpected reply {reply:?}");
18091 };
18092 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
18093 assert_eq!(
18094 sync(&handler, &incumbent, 4, Vec::new()).await,
18095 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
18096 );
18097 }
18098
18099 #[tokio::test]
18102 async fn closing_the_authority_connection_frees_sync_authority() {
18103 let supervisor = SupervisorHandle::new();
18104 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
18105 let handler = ControlHandler::new(Arc::new(Registry::default()))
18106 .with_supervisor(supervisor.clone());
18107 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
18108 sync(&handler, &first, 10, vec![head("s", 1)])
18109 .await
18110 .unwrap();
18111 handler.cleanup_connection(first.connection_id).unwrap();
18112
18113 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
18114 sync(&handler, &second, 1, vec![head("s", 1)])
18115 .await
18116 .expect("the next connection takes the released authority");
18117 }
18118
18119 #[tokio::test]
18120 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
18121 let supervisor = SupervisorHandle::new();
18122 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
18123 let handler =
18124 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
18125 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
18126 sync(&handler, &first, 10, vec![head("s", 1)])
18127 .await
18128 .unwrap();
18129 handler
18130 .handle_control_frame(
18131 &first,
18132 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
18133 )
18134 .await
18135 .unwrap();
18136 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
18137 sync(&handler, &second, 1, vec![head("s", 1)])
18138 .await
18139 .expect("GOODBYE releases authority even if the old socket remains open");
18140 }
18141
18142 #[tokio::test]
18143 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
18144 let registry = Arc::new(Registry::default());
18145 let supervisor_handle = SupervisorHandle::new();
18146 let supervisor =
18147 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
18148 .with_handle(supervisor_handle.clone())
18149 .with_daemon_incarnation("incarnation-7".to_string());
18150 supervisor
18153 .supervise_configured(
18154 ModuleSpec {
18155 carry_containment: false,
18156 module_id: OWNER.to_string(),
18157 program: PathBuf::from("/nonexistent/prefrontal-core"),
18158 args: Vec::new(),
18159 env: Vec::new(),
18160 reserved: false,
18161 reserved_prefixes: Vec::new(),
18162 protocol: ModuleProtocol::Subc,
18163 overlap: Default::default(),
18164 },
18165 false,
18166 )
18167 .unwrap();
18168 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
18169 let handler =
18170 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
18171 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
18172
18173 let ModuleControlResponseToModule::ScopeDescribe {
18175 status,
18176 daemon_incarnation,
18177 owner_synced,
18178 owner_configured,
18179 scope,
18180 ..
18181 } = describe(&handler, &reader, OWNER, "s").await
18182 else {
18183 panic!("not a describe reply");
18184 };
18185 assert_eq!(status, ScopeStatus::NotLive);
18186 assert_eq!(daemon_incarnation, "incarnation-7");
18187 assert!(!owner_synced);
18188 assert!(owner_configured);
18189 assert!(scope.is_none());
18190
18191 let ModuleControlResponseToModule::ScopeDescribe {
18194 status,
18195 owner_configured,
18196 ..
18197 } = describe(&handler, &reader, "ghost", "s").await
18198 else {
18199 panic!("not a describe reply");
18200 };
18201 assert_eq!(status, ScopeStatus::NotLive);
18202 assert!(!owner_configured);
18203
18204 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
18206 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
18207 let ModuleControlResponseToModule::ScopeDescribe {
18208 status,
18209 scope_epoch,
18210 owner_synced,
18211 scope,
18212 ..
18213 } = describe(&handler, &reader, OWNER, "s").await
18214 else {
18215 panic!("not a describe reply");
18216 };
18217 assert_eq!(status, ScopeStatus::Live);
18218 assert_eq!(scope_epoch, Some(4));
18219 assert!(owner_synced);
18220 let stamp = scope.expect("a live scope carries its stamp");
18221 assert!(
18222 stamp.owner_authorized,
18223 "prefrontal-core is the default authority"
18224 );
18225 assert_eq!(stamp.kind, ScopeKind::Head);
18226 }
18227
18228 #[tokio::test]
18229 async fn scope_authority_owners_decides_owner_authorized() {
18230 let supervisor = SupervisorHandle::new();
18231 supervisor.set_spawn_nonce("broca", "b1".to_string());
18232 let handler = ControlHandler::new(Arc::new(Registry::default()))
18233 .with_supervisor(supervisor)
18234 .with_scope_authority_owners(vec!["broca".to_string()]);
18235 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
18236 let mut gated = head("s", 1);
18237 gated.attributes.agent_id = Some("agent".to_string());
18238 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
18239 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
18240 describe(&handler, &broca, "broca", "s").await
18241 else {
18242 panic!("not a describe reply");
18243 };
18244 assert!(scope.unwrap().owner_authorized);
18245 }
18246
18247 #[tokio::test]
18251 async fn scope_ops_and_the_scopes_capability_are_advertised() {
18252 let handler = ControlHandler::new(Arc::new(Registry::default()));
18253 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
18254 let ack = hello_via_sink(
18255 &handler,
18256 &ctx,
18257 &mut rx,
18258 hello_frame("m", PROTOCOL_VERSION, 1),
18259 )
18260 .await;
18261 let ack = parse_ack(&ack);
18262 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
18263 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
18264 }
18265 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
18266
18267 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
18268 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
18269 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
18270 let reply = handler
18271 .handle_control_frame(&client, frame)
18272 .await
18273 .unwrap()
18274 .pop()
18275 .unwrap();
18276 let ClientControlResponse::ServerDescribe { capabilities, .. } =
18277 serde_json::from_slice(&reply.body).unwrap()
18278 else {
18279 panic!("not a server.describe reply");
18280 };
18281 assert!(
18282 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
18283 "{capabilities:?}"
18284 );
18285 }
18286
18287 const PLEXUS: &str = "plexus";
18290 const OTHER: &str = "other";
18291 const AFT: &str = "aft";
18292 const BROCA: &str = "broca";
18293 const MAGIC: &str = "magic-context";
18294
18295 fn nonce(module_id: &str) -> String {
18296 format!("nonce-{module_id}")
18297 }
18298
18299 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
18300 let (tx, rx) = mpsc::channel(64);
18301 (
18302 RouteCtx {
18303 connection_id: ConnectionId::new(connection),
18304 egress: FrameSink::new(tx),
18305 },
18306 rx,
18307 )
18308 }
18309
18310 struct Rig {
18314 handler: ControlHandler,
18315 forwarding: Arc<ForwardingTable>,
18316 owner: RouteCtx,
18317 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18318 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
18319 generation: u64,
18320 next_connection: u64,
18321 _supervisor: Supervisor,
18322 }
18323
18324 async fn rig() -> Rig {
18325 rig_with_flow_support(true).await
18326 }
18327
18328 async fn rig_with_flow_support(flow_support: bool) -> Rig {
18329 rig_with_scope_capabilities(flow_support, false).await
18330 }
18331
18332 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
18333 let mut provides = Vec::new();
18334 if flow_support {
18335 provides.push("flow-scopes/v1");
18336 }
18337 if run_support {
18338 provides.push("agent-run-scopes/v1");
18339 }
18340 rig_with_target_capabilities(&provides).await
18341 }
18342
18343 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
18344 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
18345 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18346 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18349 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
18350 Frame::build(
18351 FrameType::Hello,
18352 control_flags(),
18353 0,
18354 0,
18355 connection,
18356 serde_json::to_vec(&body).unwrap(),
18357 )
18358 .unwrap()
18359 }
18360
18361 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
18362 let registry = Arc::new(Registry::default());
18363 let forwarding = Arc::new(ForwardingTable::default());
18364 let supervisor_handle = SupervisorHandle::new();
18365 let supervisor =
18366 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
18367 .with_handle(supervisor_handle.clone());
18368 supervisor
18369 .supervise_configured(
18370 ModuleSpec {
18371 carry_containment: false,
18372 module_id: OWNER.to_string(),
18373 program: PathBuf::from("/nonexistent/prefrontal-core"),
18374 args: Vec::new(),
18375 env: Vec::new(),
18376 reserved: false,
18377 reserved_prefixes: Vec::new(),
18378 protocol: ModuleProtocol::Subc,
18379 overlap: Default::default(),
18380 },
18381 false,
18382 )
18383 .unwrap();
18384 for module_id in [OWNER, AFT, BROCA, MAGIC] {
18385 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
18386 }
18387 let handler =
18388 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18389 .with_supervisor(supervisor_handle);
18390 let (owner, mut owner_rx) = wide_ctx(1);
18391 hello_via_sink(
18392 &handler,
18393 &owner,
18394 &mut owner_rx,
18395 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
18396 )
18397 .await;
18398 let mut modules = BTreeMap::new();
18399 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
18400 let (ctx, mut rx) = wide_ctx(connection);
18401 let hello = hello_with_capabilities(module_id, connection, provides);
18402 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
18403 modules.insert(module_id.to_string(), (ctx, rx));
18404 }
18405 Rig {
18406 handler,
18407 forwarding,
18408 owner,
18409 _owner_rx: owner_rx,
18410 modules,
18411 generation: 0,
18412 next_connection: 100,
18413 _supervisor: supervisor,
18414 }
18415 }
18416
18417 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
18418 ScopeCarrier::new(Principal::Reserved {
18419 module_id: module_id.to_string(),
18420 })
18421 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
18422 }
18423
18424 fn session(scope_epoch: u64) -> ScopeRecord {
18427 let mut record = head("s", scope_epoch);
18428 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
18429 record.attributes.agent_id = Some("agent-1".to_string());
18430 record.attributes.delegates = true;
18431 record
18432 }
18433
18434 impl Rig {
18435 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
18436 self.generation += 1;
18437 sync(&self.handler, &self.owner, self.generation, scopes)
18438 .await
18439 .expect("the owner's sync is accepted");
18440 }
18441
18442 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18443 ScopeSelector::new(
18444 Principal::Reserved {
18445 module_id: OWNER.to_string(),
18446 },
18447 scope_ref.to_string(),
18448 scope_epoch,
18449 )
18450 }
18451
18452 fn open_frame(
18453 &mut self,
18454 opener: Option<&str>,
18455 target: &str,
18456 scope: Option<ScopeSelector>,
18457 ) -> (
18458 RouteCtx,
18459 mpsc::Receiver<crate::router::OutboundFrame>,
18460 Frame,
18461 ) {
18462 self.next_connection += 1;
18463 let (ctx, rx) = wide_ctx(self.next_connection);
18464 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
18465 busy_refusals: None,
18466 target: RouteTarget::ToolProvider {
18467 module_id: target.to_string(),
18468 },
18469 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
18470 consumer_identity: opener.map(|module_id| ConsumerIdentity {
18471 module_id: module_id.to_string(),
18472 launch_nonce: nonce(module_id),
18473 }),
18474 consumer_capabilities: None,
18475 role_versions: None,
18476 admission_facts: None,
18477 scope,
18478 })
18479 .unwrap();
18480 let frame = Frame::build(
18481 FrameType::Request,
18482 control_flags(),
18483 0,
18484 0,
18485 self.next_connection,
18486 body,
18487 )
18488 .unwrap();
18489 (ctx, rx, frame)
18490 }
18491
18492 async fn refused(
18494 &mut self,
18495 opener: Option<&str>,
18496 target: &str,
18497 scope: Option<ScopeSelector>,
18498 ) -> String {
18499 self.refusal_body(opener, target, scope).await["code"]
18500 .as_str()
18501 .unwrap()
18502 .to_string()
18503 }
18504
18505 async fn refusal_body(
18506 &mut self,
18507 opener: Option<&str>,
18508 target: &str,
18509 scope: Option<ScopeSelector>,
18510 ) -> Value {
18511 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
18512 let replies = tokio::time::timeout(
18513 Duration::from_secs(2),
18514 self.handler.handle_control_frame(&ctx, frame),
18515 )
18516 .await
18517 .expect("the open must be refused before waiting for a bind ack")
18518 .unwrap();
18519 assert_eq!(replies.len(), 1, "{replies:?}");
18520 assert_eq!(replies[0].header.ty, FrameType::Error);
18521 let (_, module_rx) = self.modules.get_mut(target).unwrap();
18522 assert!(
18523 module_rx.try_recv().is_err(),
18524 "a refused open relays nothing"
18525 );
18526 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
18527 parse_error(&replies[0])
18528 }
18529
18530 async fn relayed(
18532 &mut self,
18533 opener: Option<&str>,
18534 target: &str,
18535 scope: Option<ScopeSelector>,
18536 ) -> Relayed {
18537 self.relayed_with_legacy_root(opener, target, scope, None)
18538 .await
18539 }
18540
18541 async fn relayed_with_legacy_root(
18542 &mut self,
18543 opener: Option<&str>,
18544 target: &str,
18545 scope: Option<ScopeSelector>,
18546 legacy_root: Option<&std::path::Path>,
18547 ) -> Relayed {
18548 let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
18549 if let Some(root) = legacy_root {
18550 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
18551 body["identity"]["project_root"] = serde_json::json!(root);
18552 frame.body = serde_json::to_vec(&body).unwrap();
18553 }
18554 let handler = self.handler.clone();
18555 let task_ctx = ctx.clone();
18556 let task = tokio::spawn(async move {
18557 handler
18558 .handle_control_frame(&task_ctx, frame)
18559 .await
18560 .unwrap()
18561 });
18562 let (_, module_rx) = self.modules.get_mut(target).unwrap();
18563 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
18564 .await
18565 .expect("the target receives the relayed route.bind")
18566 .unwrap()
18567 .frame;
18568 Relayed {
18569 target: target.to_string(),
18570 client: ctx,
18571 client_rx: rx,
18572 task,
18573 bind,
18574 }
18575 }
18576
18577 async fn ack(&self, relayed: &Relayed) {
18578 let (module, _) = &self.modules[&relayed.target];
18579 self.handler
18580 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
18581 .await
18582 .unwrap();
18583 }
18584
18585 async fn bound(
18587 &mut self,
18588 opener: Option<&str>,
18589 target: &str,
18590 scope: Option<ScopeSelector>,
18591 ) -> Bound {
18592 let relayed = self.relayed(opener, target, scope).await;
18593 self.commit_relayed(relayed).await
18594 }
18595
18596 async fn bound_with_legacy_root(
18597 &mut self,
18598 opener: Option<&str>,
18599 target: &str,
18600 scope: Option<ScopeSelector>,
18601 root: &std::path::Path,
18602 ) -> Bound {
18603 let relayed = self
18604 .relayed_with_legacy_root(opener, target, scope, Some(root))
18605 .await;
18606 self.commit_relayed(relayed).await
18607 }
18608
18609 async fn commit_relayed(&self, relayed: Relayed) -> Bound {
18610 self.ack(&relayed).await;
18611 let Relayed {
18612 target,
18613 client,
18614 mut client_rx,
18615 task,
18616 bind,
18617 } = relayed;
18618 assert!(
18619 task.await.unwrap().is_empty(),
18620 "the open is answered by commit"
18621 );
18622 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
18623 Bound {
18624 target,
18625 client,
18626 client_rx,
18627 channel,
18628 epoch,
18629 bind,
18630 }
18631 }
18632
18633 fn live(&self, route: &Bound) -> bool {
18634 matches!(
18635 self.forwarding
18636 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18637 .unwrap(),
18638 DataRoute::Client(DataRouteState::Bound(_))
18639 )
18640 }
18641 }
18642
18643 struct Relayed {
18644 target: String,
18645 client: RouteCtx,
18646 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18647 task: tokio::task::JoinHandle<Vec<Frame>>,
18648 bind: Frame,
18649 }
18650
18651 struct Bound {
18652 target: String,
18653 client: RouteCtx,
18654 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18655 channel: u16,
18656 epoch: u32,
18657 bind: Frame,
18658 }
18659
18660 impl Bound {
18661 fn closed_reason(&mut self) -> RouteCloseReason {
18664 let mut reason = None;
18665 let mut goodbye = false;
18666 while let Ok(outbound) = self.client_rx.try_recv() {
18667 let frame = outbound.frame;
18668 match frame.header.ty {
18669 FrameType::Goodbye => {
18670 assert_eq!(
18671 (frame.header.channel, frame.header.epoch),
18672 (self.channel, self.epoch)
18673 );
18674 goodbye = true;
18675 }
18676 FrameType::Push => {
18677 let ClientControlPush::RouteClosed {
18678 reason: r,
18679 module_id,
18680 ..
18681 } = serde_json::from_slice(&frame.body).unwrap()
18682 else {
18683 panic!("unexpected push");
18684 };
18685 assert_eq!(module_id, self.target);
18686 reason = Some(r);
18687 }
18688 other => panic!("unexpected frame {other:?}"),
18689 }
18690 }
18691 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
18692 reason.expect("the client is told why the route closed")
18693 }
18694
18695 fn untouched(&mut self) -> bool {
18696 self.client_rx.try_recv().is_err()
18697 }
18698
18699 fn stamp(&self) -> Option<ScopeStamp> {
18700 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
18701 ModuleControlRequest::RouteBind { scope, .. } => scope,
18702 other => panic!("expected a route.bind, got {other:?}"),
18703 }
18704 }
18705 }
18706
18707 #[tokio::test]
18708 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
18709 ) {
18710 let mut rig = rig().await;
18711 let mut record = session(1);
18712 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
18713 record.child_owners = vec![Principal::Reserved {
18714 module_id: MAGIC.to_string(),
18715 }];
18716 rig.sync(vec![record]).await;
18717 let scope = || Some(rig_selector("s", Some(1)));
18718
18719 rig.bound(Some(OWNER), PLEXUS, scope()).await;
18722 rig.bound(Some(AFT), OTHER, scope()).await;
18723 rig.bound(Some(BROCA), PLEXUS, scope()).await;
18724
18725 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
18729 assert_eq!(
18730 rig.refused(opener, target, scope()).await,
18731 error_codes::SCOPE_NOT_CARRIER,
18732 "{opener:?} -> {target}"
18733 );
18734 }
18735 }
18736
18737 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18738 ScopeSelector::new(
18739 Principal::Reserved {
18740 module_id: OWNER.to_string(),
18741 },
18742 scope_ref.to_string(),
18743 scope_epoch,
18744 )
18745 }
18746
18747 #[tokio::test]
18748 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
18749 let mut rig = rig().await;
18750 rig.sync(vec![session(1)]).await;
18751 for opener in [OWNER, AFT] {
18752 assert_eq!(
18753 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
18754 .await,
18755 error_codes::SCOPE_EPOCH_REQUIRED,
18756 "{opener}"
18757 );
18758 }
18759 }
18760
18761 #[tokio::test]
18762 async fn admission_separates_not_synced_not_live_and_ended() {
18763 let mut rig = rig().await;
18764 let code = rig
18766 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18767 .await;
18768 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
18769 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
18770
18771 let ghost = ScopeSelector::new(
18773 Principal::Reserved {
18774 module_id: "ghost".to_string(),
18775 },
18776 "s".to_string(),
18777 Some(1),
18778 );
18779 assert_eq!(
18780 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
18781 error_codes::SCOPE_NOT_LIVE
18782 );
18783
18784 rig.sync(vec![session(2)]).await;
18785 assert_eq!(
18786 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
18787 .await,
18788 error_codes::SCOPE_NOT_LIVE
18789 );
18790 for epoch in [1, 3] {
18791 assert_eq!(
18792 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
18793 .await,
18794 error_codes::SCOPE_ENDED,
18795 "epoch {epoch}"
18796 );
18797 }
18798 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
18800 .await;
18801 }
18802
18803 #[tokio::test]
18804 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
18805 let mut rig = rig().await;
18806 rig.sync(vec![session(1)]).await;
18807 let route = rig
18808 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18809 .await;
18810 let stamp = route.stamp().expect("a scoped bind carries the stamp");
18811 assert_eq!(stamp.scope_ref, "s");
18812 assert_eq!(stamp.scope_epoch, 1);
18813 assert_eq!(stamp.kind, ScopeKind::Head);
18814 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
18815 assert!(stamp.attributes.delegates);
18816 assert!(stamp.owner_authorized);
18817 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18818 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
18819
18820 let (broca, mut broca_rx) = wide_ctx(50);
18823 hello_via_sink(
18824 &rig.handler,
18825 &broca,
18826 &mut broca_rx,
18827 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
18828 )
18829 .await;
18830 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
18831 .await
18832 .unwrap();
18833 let own = ScopeSelector::new(
18834 Principal::Reserved {
18835 module_id: BROCA.to_string(),
18836 },
18837 "b".to_string(),
18838 Some(1),
18839 );
18840 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
18841 assert!(!route.stamp().unwrap().owner_authorized);
18842 }
18843
18844 #[tokio::test]
18845 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
18846 let mut rig = rig().await;
18847 let mut record = head("s", 1);
18848 record.carriers = vec![carrier(AFT, None)];
18849 let flow_id = "Flow:run-7/step_2!~";
18850 record.attributes.flow_id = Some(flow_id.to_string());
18851 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
18852 .await
18853 .unwrap();
18854 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
18855 panic!("not a sync reply");
18856 };
18857 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
18858 let route = rig
18859 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18860 .await;
18861 assert!(rig.live(&route), "the stamped bind committed");
18862 let stamp = route.stamp().expect("a flow scope carries a stamp");
18863 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
18864 assert_eq!(stamp.attributes.agent_id, None);
18865 assert!(!stamp.attributes.delegates);
18866 assert!(stamp.owner_authorized);
18867 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18868 assert_eq!(unscoped.stamp(), None);
18869 }
18870
18871 #[tokio::test]
18872 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
18873 let mut rig = rig_with_flow_support(false).await;
18874 let mut record = session(1);
18875 record.attributes.flow_id = Some("flow:7".to_string());
18876 rig.sync(vec![record]).await;
18877 for opener in [OWNER, AFT] {
18878 let body = rig
18879 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18880 .await;
18881 assert_eq!(body["code"], "target_flow_unsupported");
18882 let message = body["message"].as_str().unwrap();
18883 for required in [PLEXUS, "flow-scopes/v1"] {
18884 assert!(message.contains(required), "{message}");
18885 }
18886 }
18887 }
18888
18889 #[tokio::test]
18890 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
18891 let mut rig = rig_with_flow_support(true).await;
18892 let mut record = session(1);
18893 record.attributes.flow_id = Some("flow:7".to_string());
18894 rig.sync(vec![record]).await;
18895 for opener in [OWNER, AFT] {
18896 let route = rig
18897 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18898 .await;
18899 assert!(rig.live(&route));
18900 assert_eq!(
18901 route.stamp().unwrap().attributes.flow_id.as_deref(),
18902 Some("flow:7")
18903 );
18904 }
18905 }
18906
18907 #[tokio::test]
18908 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
18909 use std::future::Future;
18910
18911 let mut rig = rig_with_flow_support(true).await;
18912 let mut record = session(1);
18913 record.attributes.flow_id = Some("flow:7".to_string());
18914 rig.sync(vec![record]).await;
18915 let (client, mut client_rx, frame) =
18916 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
18917 for _ in 0..64 {
18920 client.egress.try_send(route_bind_ack(1)).unwrap();
18921 }
18922 let handler = rig.handler.clone();
18923 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18924 std::future::poll_fn(|cx| {
18925 assert!(open.as_mut().poll(cx).is_pending());
18926 std::task::Poll::Ready(())
18927 })
18928 .await;
18929 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18930
18931 let old_connection = rig.modules[PLEXUS].0.connection_id;
18932 rig.handler.cleanup_connection(old_connection).unwrap();
18933 let (replacement, mut replacement_rx) = wide_ctx(200);
18934 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
18935 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18936 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18937 let hello = Frame::build(
18938 FrameType::Hello,
18939 control_flags(),
18940 0,
18941 0,
18942 200,
18943 serde_json::to_vec(&body).unwrap(),
18944 )
18945 .unwrap();
18946 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
18947
18948 client_rx.try_recv().unwrap();
18949 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18950 .await
18951 .expect("the replacement is refused without waiting for a bind ack")
18952 .unwrap();
18953 assert_eq!(replies.len(), 1);
18954 let body = parse_error(&replies[0]);
18955 assert_eq!(body["code"], "target_flow_unsupported");
18956 for required in [PLEXUS, "flow-scopes/v1"] {
18957 assert!(body["message"].as_str().unwrap().contains(required));
18958 }
18959 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
18960 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18961 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18962 assert!(rig
18963 .handler
18964 .registry
18965 .get_module_by_connection(replacement.connection_id)
18966 .unwrap()
18967 .is_some());
18968 }
18969
18970 #[tokio::test]
18971 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
18972 use std::future::Future;
18973 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
18974 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18975 const NEXT_HASH: &str =
18976 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18977 let mut rig = rig().await;
18978 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
18979 record.carriers.push(ScopeCarrier::new(Principal::Direct));
18980 rig.sync(vec![record.clone()]).await;
18981 let (client, mut rx, frame) = rig.open_frame(
18982 None,
18983 PLEXUS,
18984 Some(
18985 rig.selector("s", Some(1))
18986 .with_bind_proof(Some(PROOF.into())),
18987 ),
18988 );
18989 for _ in 0..64 {
18992 client.egress.try_send(route_bind_ack(1)).unwrap();
18993 }
18994 let handler = rig.handler.clone();
18995 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18996 std::future::poll_fn(|cx| {
18997 assert!(open.as_mut().poll(cx).is_pending());
18998 std::task::Poll::Ready(())
18999 })
19000 .await;
19001 record.bind_proof_sha256 = Some(NEXT_HASH.into());
19002 rig.sync(vec![record]).await;
19003 rx.try_recv().unwrap();
19004 let replies = tokio::time::timeout(Duration::from_secs(2), open)
19005 .await
19006 .unwrap()
19007 .unwrap();
19008 assert_eq!(
19009 parse_error(&replies[0])["code"],
19010 error_codes::SCOPE_BIND_PROOF_MISMATCH
19011 );
19012 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
19013 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
19014 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19015 }
19016
19017 #[tokio::test]
19018 async fn scope_bind_proof_reserved_openers_ignore_proof() {
19019 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
19020 let mut rig = rig().await;
19021 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
19022 .await;
19023 for opener in [OWNER, AFT] {
19024 for proof in [None, Some("malformed+proof=")] {
19025 let route = rig
19026 .bound(
19027 Some(opener),
19028 PLEXUS,
19029 Some(
19030 rig.selector("s", Some(1))
19031 .with_bind_proof(proof.map(str::to_string)),
19032 ),
19033 )
19034 .await;
19035 assert!(rig.live(&route));
19036 }
19037 }
19038 }
19039
19040 #[tokio::test]
19041 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
19042 use std::future::Future;
19043 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
19044 const NEXT_HASH: &str =
19045 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
19046 let mut rig = rig().await;
19047 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
19048 .await;
19049 let (client, mut rx, frame) =
19050 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
19051 for _ in 0..64 {
19052 client.egress.try_send(route_bind_ack(1)).unwrap();
19053 }
19054 let handler = rig.handler.clone();
19055 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
19056 std::future::poll_fn(|cx| {
19057 assert!(open.as_mut().poll(cx).is_pending());
19058 std::task::Poll::Ready(())
19059 })
19060 .await;
19061 rig.sync(vec![
19062 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
19063 ])
19064 .await;
19065 rx.try_recv().unwrap();
19066 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19067 let bind = std::future::poll_fn(|cx| {
19068 assert!(open.as_mut().poll(cx).is_pending());
19069 module_rx.poll_recv(cx)
19070 })
19071 .await
19072 .unwrap()
19073 .frame;
19074 handler
19075 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
19076 .await
19077 .unwrap();
19078 let replies = tokio::time::timeout(Duration::from_secs(2), open)
19079 .await
19080 .unwrap()
19081 .unwrap();
19082 assert_eq!(replies.len(), 1);
19083 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
19086 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19087 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
19088 let route = rig
19089 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
19090 .await;
19091 assert!(rig.live(&route));
19092 }
19093
19094 #[tokio::test]
19095 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
19096 {
19097 let mut rig = rig_with_flow_support(false).await;
19098 rig.sync(vec![session(1)]).await;
19099 let route = rig
19100 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19101 .await;
19102 assert!(rig.live(&route));
19103 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
19104 let mut record = session(1);
19105 record.attributes.flow_id = Some("flow:7".to_string());
19106 rig.sync(vec![record]).await;
19107 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
19108 assert!(rig.live(&unscoped));
19109 assert_eq!(unscoped.stamp(), None);
19110 }
19111
19112 #[tokio::test]
19113 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
19114 let mut rig = rig().await;
19115 let mut record = session(1);
19116 record.attributes.flow_id = Some("flow:7".to_string());
19117 rig.sync(vec![record.clone()]).await;
19118 let before = rig.forwarding.published_scope_tag(OWNER, "s");
19119 let mut owner_route = rig
19120 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
19121 .await;
19122 let mut carrier_route = rig
19123 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
19124 .await;
19125 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
19126 rig.sync(vec![record.clone()]).await;
19127 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
19128 assert!(rig.live(&owner_route) && owner_route.untouched());
19129 assert!(rig.live(&carrier_route) && carrier_route.untouched());
19130
19131 record.attributes.flow_id = Some("flow:8".to_string());
19132 rig.sync(vec![record]).await;
19133 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
19134 let before = before.unwrap();
19135 assert_eq!(after.scope_epoch, before.scope_epoch);
19136 assert!(after.version > before.version);
19137 for route in [&mut owner_route, &mut carrier_route] {
19138 assert!(!rig.live(route));
19139 assert_eq!(
19140 route.closed_reason(),
19141 RouteCloseReason::ScopeDelegationChanged
19142 );
19143 }
19144 assert!(rig.live(&unscoped) && unscoped.untouched());
19145 for target in [PLEXUS, OTHER] {
19148 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
19149 let goodbye = module_rx
19150 .try_recv()
19151 .expect("the provider sees the drain")
19152 .frame;
19153 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19154 assert!(module_rx.try_recv().is_err());
19155 }
19156 let rebound = rig
19157 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19158 .await;
19159 assert_eq!(
19160 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
19161 Some("flow:8")
19162 );
19163 }
19164
19165 #[tokio::test]
19170 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
19171 let mut rig = rig().await;
19172 rig.sync(vec![session(1)]).await;
19173 let mut cotenant = rig
19174 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
19175 .await;
19176
19177 let mut changed = session(1);
19178 changed.child_owners.push(Principal::Reserved {
19179 module_id: MAGIC.to_string(),
19180 });
19181 let mut ended = None;
19182 for (code, next) in [
19183 (error_codes::SCOPE_CHANGED, vec![changed]),
19184 (error_codes::SCOPE_ENDED, Vec::new()),
19185 ] {
19186 let relayed = rig
19187 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19188 .await;
19189 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
19190 ended = Some(next.is_empty());
19191 rig.sync(next).await;
19192 rig.ack(&relayed).await;
19193 let replies = relayed.task.await.unwrap();
19194 assert_eq!(replies.len(), 1, "{replies:?}");
19195 assert_eq!(parse_error(&replies[0])["code"], code);
19196 assert_eq!(
19197 subc_protocol::error_codes::is_retryable_route_open(code),
19198 code == error_codes::SCOPE_CHANGED
19199 );
19200 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19201 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19203 let mut goodbyes = Vec::new();
19206 while let Ok(outbound) = plexus_rx.try_recv() {
19207 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
19208 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
19209 }
19210 assert!(
19211 goodbyes.contains(&(bind_channel, bind_epoch)),
19212 "{goodbyes:?}"
19213 );
19214 assert!(rig
19215 .handler
19216 .registry
19217 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
19218 .unwrap()
19219 .is_some());
19220 }
19221 assert_eq!(ended, Some(true));
19222 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
19225 }
19226
19227 #[tokio::test]
19230 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
19231 struct Case {
19232 name: &'static str,
19233 change: fn(&mut ScopeRecord),
19234 closed: [Option<RouteCloseReason>; 4],
19237 }
19238 use RouteCloseReason::*;
19239 let cases = [
19240 Case {
19241 name: "a carrier entry removed",
19242 change: |r| {
19243 r.carriers.retain(|c| {
19244 c.principal
19245 != Principal::Reserved {
19246 module_id: AFT.to_string(),
19247 }
19248 })
19249 },
19250 closed: [None, Some(ScopeCarrierRemoved), None, None],
19251 },
19252 Case {
19253 name: "a target removed from a carrier",
19254 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
19255 closed: [None, None, None, Some(ScopeCarrierRemoved)],
19256 },
19257 Case {
19258 name: "a bare carrier narrowed to targets",
19259 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
19260 closed: [None, Some(ScopeCarrierRemoved), None, None],
19261 },
19262 Case {
19263 name: "delegates turned off",
19264 change: |r| r.attributes.delegates = false,
19265 closed: [Some(ScopeDelegationChanged); 4],
19266 },
19267 Case {
19268 name: "agent_id changed",
19269 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
19270 closed: [Some(ScopeDelegationChanged); 4],
19271 },
19272 Case {
19273 name: "a carrier added, child owners changed, the record re-sent",
19274 change: |r| {
19275 r.carriers.push(carrier(MAGIC, None));
19276 r.child_owners.push(Principal::Reserved {
19277 module_id: MAGIC.to_string(),
19278 });
19279 },
19280 closed: [None; 4],
19281 },
19282 Case {
19283 name: "a target added",
19284 change: |r| {
19285 r.carriers[1]
19286 .targets
19287 .as_mut()
19288 .unwrap()
19289 .push("third".to_string())
19290 },
19291 closed: [None; 4],
19292 },
19293 Case {
19294 name: "delegates turned on",
19295 change: |r| r.attributes.delegates = true,
19296 closed: [None; 4],
19297 },
19298 ];
19299 for case in cases {
19300 let mut rig = rig().await;
19301 rig.sync(vec![session(1)]).await;
19302 let scope = || Some(rig_selector("s", Some(1)));
19303 let mut routes = [
19304 rig.bound(Some(OWNER), PLEXUS, scope()).await,
19305 rig.bound(Some(AFT), PLEXUS, scope()).await,
19306 rig.bound(Some(BROCA), PLEXUS, scope()).await,
19307 rig.bound(Some(BROCA), OTHER, scope()).await,
19308 ];
19309 let mut record = session(1);
19310 (case.change)(&mut record);
19311 rig.sync(vec![record]).await;
19312 for (index, expected) in case.closed.iter().enumerate() {
19313 let route = &mut routes[index];
19314 match expected {
19315 Some(reason) => {
19316 assert!(!rig.live(route), "{}: route {index} still live", case.name);
19317 assert_eq!(
19318 route.closed_reason(),
19319 *reason,
19320 "{}: route {index}",
19321 case.name
19322 );
19323 }
19324 None => {
19325 assert!(rig.live(route), "{}: route {index} closed", case.name);
19326 assert!(
19327 route.untouched(),
19328 "{}: route {index} was told something",
19329 case.name
19330 );
19331 }
19332 }
19333 }
19334 }
19335 }
19336
19337 #[tokio::test]
19338 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
19339 for next in [Vec::new(), vec![session(2)]] {
19341 let mut rig = rig().await;
19342 rig.sync(vec![session(1)]).await;
19343 let mut route = rig
19344 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19345 .await;
19346 rig.sync(next).await;
19347 assert!(!rig.live(&route));
19348 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
19349 }
19350
19351 let mut rig = rig().await;
19354 let mut child = session(1);
19355 child.scope_ref = "child".to_string();
19356 child.kind = ScopeKind::Worker;
19357 child.parent = Some(ScopeParent::new(
19358 Principal::Reserved {
19359 module_id: OWNER.to_string(),
19360 },
19361 "s".to_string(),
19362 1,
19363 ));
19364 rig.sync(vec![session(1), child.clone()]).await;
19365 let mut child_route = rig
19366 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
19367 .await;
19368 assert_eq!(
19369 child_route.stamp().unwrap().parent_state,
19370 Some(ParentState::Linked)
19371 );
19372 rig.sync(vec![child]).await;
19373 assert!(!rig.live(&child_route));
19374 assert_eq!(
19375 child_route.closed_reason(),
19376 RouteCloseReason::ScopeParentEnded
19377 );
19378 }
19379
19380 #[tokio::test]
19381 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
19382 ) {
19383 let mut rig = rig().await;
19384 rig.sync(vec![session(1)]).await;
19385 let mut route = rig
19386 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19387 .await;
19388 let before = rig.forwarding.published_scope_tag(OWNER, "s");
19389 rig.sync(vec![session(1)]).await;
19390 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
19391 assert!(rig.live(&route) && route.untouched());
19392
19393 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
19397 .forwarding
19398 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
19399 .unwrap()
19400 else {
19401 panic!("the route is bound");
19402 };
19403 binding.flow.acquire_tagged(9, false).await.unwrap();
19404 let mut widened = session(1);
19405 widened.carriers.push(carrier(MAGIC, None));
19406 rig.sync(vec![widened]).await;
19407 assert!(rig.live(&route) && route.untouched());
19408 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19409 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
19410 assert_eq!(binding.flow.in_flight(), 1);
19413 binding
19414 .flow
19415 .acquire_tagged(10, false)
19416 .await
19417 .expect("the flow is still open");
19418 }
19419
19420 #[tokio::test]
19423 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
19424 let mut rig = rig().await;
19425 rig.sync(vec![session(1)]).await;
19426 let mut on_incumbent = rig
19427 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19428 .await;
19429
19430 let (candidate, _candidate_rx) = wide_ctx(9);
19433 let registration = rig
19434 .handler
19435 .registry
19436 .register_candidate_with_control_ops(
19437 manifest(PLEXUS, PROTOCOL_VERSION),
19438 PROTOCOL_VERSION,
19439 candidate.connection_id,
19440 module_baseline_control_ops(),
19441 )
19442 .unwrap();
19443 rig.forwarding
19444 .register_candidate_module_connection(
19445 candidate.connection_id,
19446 PLEXUS.to_string(),
19447 PROTOCOL_VERSION,
19448 manifest_concurrency(®istration.manifest),
19449 candidate.egress.clone(),
19450 )
19451 .unwrap();
19452 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
19453 rig.handler
19454 .registry
19455 .promote_candidate(PLEXUS)
19456 .unwrap()
19457 .unwrap();
19458 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
19459
19460 rig.sync(Vec::new()).await;
19461 assert!(!rig.live(&on_incumbent));
19462 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
19463 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19464 let goodbye = incumbent_rx
19465 .try_recv()
19466 .expect("the superseded endpoint is told")
19467 .frame;
19468 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19469 }
19470 }
19471}
19472
19473#[cfg(test)]
19474mod concurrency_default_exposure_tests {
19475 use super::*;
19476
19477 fn hello_body(role_json: &str) -> Vec<u8> {
19478 format!(
19479 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
19480 )
19481 .into_bytes()
19482 }
19483
19484 fn manifest_from(body: &[u8]) -> ModuleManifest {
19485 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
19486 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
19487 .expect("manifest parses")
19488 }
19489
19490 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
19491
19492 #[test]
19493 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
19494 let body = hello_body(&format!(
19495 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
19496 ));
19497 let manifest = manifest_from(&body);
19498 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
19501 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
19502 }
19503
19504 #[test]
19505 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
19506 let body = hello_body(&format!(
19507 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
19508 ));
19509 let manifest = manifest_from(&body);
19510 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
19511 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19512 }
19513
19514 #[test]
19515 fn non_management_roles_are_never_reported() {
19516 let body = hello_body(
19517 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
19518 );
19519 let manifest = manifest_from(&body);
19520 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19521 }
19522}