1#[path = "control/list_diagnostics.rs"]
2pub(crate) mod list_diagnostics;
3
4use std::{
5 collections::{BTreeMap, BTreeSet, HashMap, HashSet},
6 fmt,
7 net::SocketAddr,
8 path::{Path, PathBuf},
9 sync::{Arc, Mutex, RwLock},
10 time::{Duration, Instant as StdInstant},
11};
12
13use serde::{Deserialize, Serialize};
14use subc_control::{
15 ops, CapabilityRequirementStatus, CatalogEntry, ClientControlPush, ClientControlRequest,
16 ClientControlResponse, ConsumerIdentity, DaemonBuildProvenance, DaemonObservedProcess,
17 ModuleDeclaredProvenance, ModuleProtocol, NotReadyReason, PendingReloadVerdict, PollKind,
18 ReloadPathAgreement, ReloadPathUnavailableReason, RouteCloseReason, SpawnCursor,
19 StderrCaptureState, StderrTail, StderrTailEntry, SupervisorDaemonProvenance, SupervisorEntry,
20 SupervisorHealthEntry, SupervisorModuleProvenance, SupervisorObservedProcess,
21 SupervisorRescanResult, SupervisorRoute, SupervisorRouteConsumer, SupervisorRouteModule,
22};
23use subc_protocol::{
24 call_mode::{derive_call_mode, CALL_MODES_CAPABILITY},
25 error_codes,
26 manifest::{
27 validate_hello_capability_grammar, validate_hello_confirm_operations,
28 validate_hello_display_name, validate_hello_event_declarations,
29 validate_hello_flow_operations, validate_hello_rich_ui_descriptor,
30 validate_hello_role_declarations, validate_hello_self_signal_declarations,
31 CapabilityDeclarations, CapabilityNeed, Concurrency, ManifestProvenance, ModuleManifest,
32 ProviderRole,
33 },
34 peer::{ObservedPeer, RouteBindPeer, OBSERVED_PEER_CAPABILITY, OPERATOR_APP_CAPABILITY},
35 scope::{
36 ScopeEnd, ScopeRecord, ScopeRecordOutcome, ScopeRecordResult, ScopeSelector,
37 CAP_ROUTE_ROLE_VERSIONS_V1, CAP_SCOPES_V1, SCOPE_APPLY_OP, SCOPE_DESCRIBE_OP,
38 SCOPE_SYNC_OP,
39 },
40 session::{
41 validate_role_versions, HealthReport, ModuleControlPush, ModuleControlRequest,
42 ModuleControlRequestFromModule, ModuleControlResponse, ModuleControlResponseToModule,
43 OperatorConfirmRequest, MODULE_CONTROL_OP_HEALTH_CHECK, MODULE_TO_SUBC_OP_CATALOG_UPDATE,
44 ROLE_VERSIONS_FIELD,
45 },
46 BindIdentity, ErrorBody, Flags, FrameType, ModuleHelloAckBody, ModuleHelloBody, Principal,
47 Priority, RouteTarget, PROTOCOL_VERSION,
48};
49use tokio::time::{timeout_at, Instant};
50use tracing::{debug, info, instrument::WithSubscriber, warn};
51
52use crate::{
53 caller_identity::CallerIdentity,
54 capability_requirements::{
55 log_duplicate_claim_events, log_manifest_change, log_requirement_events,
56 CapabilityRequirementEvaluator, CapabilityVerdict, DuplicateClaimSource, RegisteredModule,
57 RequirementStatus, RuntimeModule,
58 },
59 child_roster::DaemonShutdownFlag,
60 daemon_config::{PrivilegedOperatorPrompt, RestartRequiredSection},
61 forwarding::{
62 CloseReason, EndpointRoute, ForwardingError, ForwardingTable, GoodbyeTarget,
63 ModuleControlRpcCompletion, ModuleControlRpcOutcome, ModuleEndpointId,
64 PendingModuleControlRpc, RouteBindRelayOutcome, RoutePollSnapshot, RouteRelease,
65 },
66 observability::{
67 ROUTE_OPEN_REFUSED_DECLARED_NOT_READY, ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED,
68 },
69 privileged_prompt::{prompt_text, PrivilegedOperation},
70 provenance::{
71 process_start_time, spawned_file_identity, ExecutableIdentityProbe, SpawnedFileIdentity,
72 },
73 registry::{ChannelState, ConnectionId, RegistrationEndReason, Registry, RegistryError},
74 router::{daemon_error_flags, RouteCtx, RouterError},
75 scopes::{HelloLaunchNonces, ScopeRecords, ScopeTable},
76 server::MAX_PENDING_ROUTE_BINDS_PER_TARGET,
77 stderr_tail::{CaptureState, TailEntry},
78 supervise::{
79 validate_spec, ModuleProcessLiveness, ReservedHelloRejection, ReservedHelloVerification,
80 SpawnSubscribeRefusal, SupervisorHandle, SwapHelloAdmission,
81 },
82 ConnectedClients, DaemonCounters, Frame, ProjectRootId, Supervisor,
83};
84
85pub const MIN_SUPPORTED_VERSION: u8 = PROTOCOL_VERSION;
91
92const CAP_MANIFEST_REGISTRATION: &str = "manifest_registration_v1";
93const CAP_CHANNEL_LIFECYCLE: &str = "channel_lifecycle_v1";
94const CAP_PING_PONG: &str = "ping_pong_v1";
95const CAP_SESSION_ATTACH: &str = "session_attach_v1";
96const CAP_ADMISSION_FACTS_RELAY: &str = "admission_facts_relay_v1";
97
98const SUBC_CONTROL_OPS: &[&str] = &[
99 ops::DAEMON_SHUTDOWN_INTENT,
100 ops::ACTIVITY_RECORD,
101 ops::ACTIVITY_QUERY,
102 ops::ACTIVITY_SUBSCRIBE,
103 ops::SERVER_DESCRIBE,
104 ops::CATALOG_LIST,
105 ops::ROUTE_OPEN,
106 ops::ROUTE_POLL,
107 ops::ROUTE_CLOSING,
108 ops::ROUTE_CLOSED,
109 ops::SUPERVISOR_LIST,
110 ops::SUPERVISOR_RESTART,
111 ops::SUPERVISOR_SWAP,
112 ops::SUPERVISOR_RELOAD,
113 ops::SUPERVISOR_RESCAN,
114 ops::SUPERVISOR_RELEASE_RESERVED,
115 ops::SUPERVISOR_SET_ENABLED,
116 ops::SUPERVISOR_HEALTH_PROBE,
117 ops::SUPERVISOR_HEALTH,
118 ops::SUPERVISOR_STDERR_TAIL,
119 ops::SUPERVISOR_TERMINALS,
120 ops::SUPERVISOR_ROUTES,
121 ops::SUPERVISOR_PROVENANCE,
122 ops::SUPERVISOR_SPAWN_SNAPSHOT,
123 ops::SUPERVISOR_SPAWN_SUBSCRIBE,
124];
125
126const SUBC_UNADVERTISED_CONTROL_OPS: &[&str] = &[ops::SUPERVISOR_RESCAN_CHECKED];
133
134mod agent_commands;
135mod rescan_checked;
136use rescan_checked::{RescanMode, RescanPlan};
137
138const MODULE_TO_SUBC_CONTROL_OPS: &[&str] = &[
139 MODULE_TO_SUBC_OP_CATALOG_UPDATE,
140 "supervisor.live_roots",
141 "supervisor.agent_commands",
142 SCOPE_SYNC_OP,
143 SCOPE_APPLY_OP,
144 SCOPE_DESCRIBE_OP,
145 "operator.confirm",
146];
147
148const MODULE_TO_SUBC_UNADVERTISED_OPS: &[&str] = &[];
152
153const MODULE_BASELINE_CONTROL_OPS: &[&str] = &["route.bind", "route.status"];
154
155pub const DEFAULT_ROUTE_BIND_RELAY_TIMEOUT: Duration = Duration::from_secs(12);
163
164pub const DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD: u32 = 3;
179
180pub const DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN: Duration = Duration::from_secs(20);
198
199const DEFAULT_HEALTH_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
200const SLOW_CONTROL_DISPATCH_THRESHOLD: Duration = Duration::from_secs(1);
201
202fn reload_verdict(
203 configured: &Path,
204 spawned_from: Option<&Path>,
205 image: subc_control::RunningImageAgreement,
206) -> PendingReloadVerdict {
207 let path = match spawned_from {
208 Some(spawned_from) if reload_paths_match(configured, spawned_from) => {
209 ReloadPathAgreement::Match
210 }
211 Some(spawned_from) => ReloadPathAgreement::Mismatch {
212 configured: configured.to_path_buf(),
213 spawned_from: spawned_from.to_path_buf(),
214 },
215 None => ReloadPathAgreement::Unavailable {
216 reason: if matches!(
217 image,
218 subc_control::RunningImageAgreement::Unavailable {
219 reason: subc_control::RunningImageUnavailableReason::NotRunning
220 }
221 ) {
222 ReloadPathUnavailableReason::NotRunning
223 } else {
224 ReloadPathUnavailableReason::SpawnedPathUnavailable
225 },
226 },
227 };
228 PendingReloadVerdict::new(path, image)
229}
230
231fn launch_spec_agreement(
232 configured: &crate::supervise::ModuleSpec,
233 running: bool,
234 spawned: Option<&crate::supervise::LaunchSpecDigest>,
235) -> subc_control::LaunchSpecAgreement {
236 use subc_control::{LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason};
237 if !running {
238 return Agreement::Unavailable {
239 reason: Reason::NotRunning,
240 };
241 }
242 match spawned {
243 None => Agreement::Unavailable {
244 reason: Reason::SpawnedSpecUnavailable,
245 },
246 Some(digest) if *digest == crate::supervise::LaunchSpecDigest::from_spec(configured) => {
247 Agreement::Match
248 }
249 Some(_) => Agreement::Mismatch,
250 }
251}
252
253fn reload_paths_match(configured: &Path, spawned_from: &Path) -> bool {
254 if configured == spawned_from {
255 return true;
256 }
257 #[cfg(windows)]
258 {
259 match (
265 std::fs::canonicalize(configured),
266 std::fs::canonicalize(spawned_from),
267 ) {
268 (Ok(configured), Ok(spawned_from)) => configured == spawned_from,
269 _ => false,
270 }
271 }
272 #[cfg(not(windows))]
273 {
274 false
275 }
276}
277
278#[derive(Clone)]
279struct DaemonProvenanceFacts {
280 build: DaemonBuildProvenance,
281 pid: Option<u32>,
282 started_at_ms: Option<u64>,
283 start_clock: Option<crate::clock::StartClock>,
284 executable_path: Option<PathBuf>,
285 executable_identity: Option<SpawnedFileIdentity>,
286 process_start_time: Option<u64>,
287 probe: ExecutableIdentityProbe,
288}
289
290impl Default for DaemonProvenanceFacts {
291 fn default() -> Self {
292 Self {
293 build: DaemonBuildProvenance {
294 build_git_sha: None,
295 build_lock_digest: None,
296 },
297 pid: None,
298 started_at_ms: None,
299 start_clock: None,
300 executable_path: None,
301 executable_identity: None,
302 process_start_time: None,
303 probe: ExecutableIdentityProbe::default(),
304 }
305 }
306}
307
308#[derive(Debug, Clone)]
309struct SupervisorRescanContext {
310 supervisor: Supervisor,
311 config_path: PathBuf,
312 configured_port: Option<u16>,
313 storage_config: Option<crate::daemon_config::StorageConfig>,
314 admission_facts_carrier_module_id: Option<String>,
315 admission_facts_targets: Option<Vec<String>>,
316 scope_authority_owners: Vec<String>,
317 federated_scope_owner: String,
318 privileged_operator_prompt: PrivilegedOperatorPrompt,
319 generation: Arc<std::sync::atomic::AtomicU64>,
325}
326
327const ROUTE_OPEN_NOT_SERVING_REASONS: &[&str] = &[
348 "reloading",
349 "supervisor_not_live",
350 "registration_not_active",
351 "no_forwarding_connection",
352 "relay_send_failed",
353];
354
355#[derive(Clone)]
357pub struct ControlHandler {
358 registry: Arc<Registry>,
359 activity: Option<crate::activity::ActivityLedger>,
360 activity_streams: Arc<Mutex<HashMap<(ConnectionId, u64), tokio::task::AbortHandle>>>,
361 forwarding: Arc<ForwardingTable>,
362 process_liveness: Option<Arc<dyn ModuleProcessLiveness>>,
363 supervisor: SupervisorHandle,
364 pub(crate) resource_tracker: Arc<crate::resource_totals::Tracker>,
365 agent_declarations: agent_commands::Queues,
366 list_admission: Arc<tokio::sync::Semaphore>,
367 subc_capabilities: Arc<[String]>,
368 route_bind_relay_timeout: Duration,
372 route_bind_relay_timeouts: BTreeMap<String, Duration>,
376 route_bind_breakers: RouteBindBreakers,
379 route_bind_concurrency: RouteBindConcurrency,
382 route_outages: Arc<crate::route_outage::RouteOutageTracker>,
387 route_bind_breaker_threshold: u32,
389 route_bind_breaker_cooldown: Duration,
391 health_probe_timeout: Duration,
392 storage_config: Option<crate::daemon_config::StorageConfig>,
395 machine_id: Option<crate::machine_id::MachineId>,
399 admission_facts_carrier_module_id: Option<String>,
400 admission_facts_targets: Option<Vec<String>>,
401 scopes: Arc<RwLock<ScopeTable>>,
405 scope_deadline_added: Arc<tokio::sync::Notify>,
409 scope_authority_owners: Vec<String>,
412 federated_scope_owner: String,
413 privileged_operator_prompt: PrivilegedOperatorPrompt,
415 caller_endpoints: Arc<Mutex<HashMap<ConnectionId, (SocketAddr, SocketAddr)>>>,
418 build_class_warned: Arc<Mutex<HashSet<ConnectionId>>>,
419 observed_peers: Arc<Mutex<HashMap<ConnectionId, ObservedPeer>>>,
420 local_connections: Arc<Mutex<HashSet<ConnectionId>>>,
421 pub(crate) operator_app_policy: crate::operator_apps::OperatorAppPolicy,
422 pub(crate) operator_apps:
423 Arc<Mutex<HashMap<ConnectionId, crate::operator_apps::OperatorAppStamp>>>,
424 pub(crate) launch_table: std::sync::Arc<crate::app_open::LaunchTable>,
429 pub(crate) app_open_launch_lock: std::sync::Arc<tokio::sync::Mutex<()>>,
435 #[cfg(test)]
437 pub(crate) app_open_test_platform: Option<crate::operator_apps::AppOpenPlatform>,
438 #[cfg(test)]
439 pub(crate) app_open_test_launch: Option<fn() -> Result<u32, crate::app_open::AppOpenRefusal>>,
440 #[allow(dead_code)]
444 #[cfg(windows)]
445 pub(crate) windows_verify: Option<crate::app_open::WindowsVerifyFn>,
446 #[allow(dead_code)]
447 #[cfg(windows)]
448 pub(crate) windows_flash: Option<crate::app_open::WindowsFlashFn>,
449 #[allow(dead_code)]
453 #[cfg(windows)]
454 pub(crate) windows_track: Option<crate::app_open::WindowsTrackFn>,
455 #[cfg(all(test, windows))]
458 pub(crate) windows_spawn_attempts: Arc<std::sync::atomic::AtomicU64>,
459 #[cfg(test)]
460 caller_lookup: Option<TestCallerLookup>,
461 hello_launch_nonces: Arc<Mutex<HelloLaunchNonces>>,
464 rescan: Option<SupervisorRescanContext>,
465 connected_clients: ConnectedClients,
466 counters: DaemonCounters,
467 local_endpoint_unavailable: Option<String>,
470 capability_evaluator: Arc<CapabilityRequirementEvaluator>,
471 daemon_shutdown: DaemonShutdownFlag,
476 daemon_provenance: DaemonProvenanceFacts,
477 #[cfg(test)]
478 control_dispatch_delay: Option<Duration>,
479 #[cfg(test)]
480 provenance_probe_override: Option<subc_control::RunningImageAgreement>,
481 #[cfg(test)]
485 route_bind_relay_sent_hold: Option<Arc<tokio::sync::Barrier>>,
486 #[cfg(test)]
492 hello_registry_to_forwarding_hold: Option<Arc<std::sync::Barrier>>,
493}
494
495impl fmt::Debug for ControlHandler {
496 fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
497 f.debug_struct("ControlHandler")
498 .field("registry", &self.registry)
499 .field("forwarding", &self.forwarding)
500 .field("process_liveness", &self.process_liveness.is_some())
501 .field("supervisor", &self.supervisor)
502 .field("subc_capabilities", &self.subc_capabilities)
503 .finish()
504 }
505}
506
507struct RouteOpenRequest {
508 target: RouteTarget,
509 identity: BindIdentity,
510 consumer_identity: Option<ConsumerIdentity>,
511 consumer_capabilities: Option<Vec<String>>,
512 role_versions: Option<BTreeMap<String, String>>,
513 busy_refusals: Option<bool>,
514 admission_facts: Option<serde_json::Value>,
515 scope: Option<ScopeSelector>,
516}
517
518#[derive(Clone, Copy)]
519enum RouteOpenPhase {
520 Admission,
521 ProjectRoot,
522 EgressReservation,
523 BindSend,
524 BindAckWait,
525}
526
527struct RouteOpenTimings {
531 module_id: String,
532 principal: String,
533 admission: Duration,
534 project_root: Duration,
535 egress_reservation: Duration,
536 bind_send: Duration,
537 bind_ack_wait: Duration,
538 phase: Option<RouteOpenPhase>,
539 phase_started_at: Instant,
540}
541
542impl RouteOpenTimings {
543 fn new() -> Self {
544 Self {
545 module_id: String::new(),
546 principal: "unresolved".to_string(),
548 admission: Duration::ZERO,
549 project_root: Duration::ZERO,
550 egress_reservation: Duration::ZERO,
551 bind_send: Duration::ZERO,
552 bind_ack_wait: Duration::ZERO,
553 phase: Some(RouteOpenPhase::Admission),
554 phase_started_at: Instant::now(),
555 }
556 }
557
558 fn finish(&mut self) {
559 let elapsed = self.phase_started_at.elapsed();
560 match self.phase.take() {
561 Some(RouteOpenPhase::Admission) => self.admission += elapsed,
562 Some(RouteOpenPhase::ProjectRoot) => self.project_root += elapsed,
563 Some(RouteOpenPhase::EgressReservation) => self.egress_reservation += elapsed,
564 Some(RouteOpenPhase::BindSend) => self.bind_send += elapsed,
565 Some(RouteOpenPhase::BindAckWait) => self.bind_ack_wait += elapsed,
566 None => {}
567 }
568 }
569
570 fn enter(&mut self, phase: RouteOpenPhase) {
571 self.finish();
572 self.phase = Some(phase);
573 self.phase_started_at = Instant::now();
574 }
575}
576
577struct RouteBindReservationGuard {
578 forwarding: Arc<ForwardingTable>,
579 endpoint: ModuleEndpointId,
580 relay_corr: u64,
581 armed: bool,
582}
583
584struct ModuleControlRpcGuard {
585 forwarding: Arc<ForwardingTable>,
586 endpoint: ModuleEndpointId,
587 corr: u64,
588 armed: bool,
589}
590
591impl ModuleControlRpcGuard {
592 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, corr: u64) -> Self {
593 Self {
594 forwarding,
595 endpoint,
596 corr,
597 armed: true,
598 }
599 }
600
601 fn disarm(&mut self) {
602 self.armed = false;
603 }
604}
605
606impl Drop for ModuleControlRpcGuard {
607 fn drop(&mut self) {
608 if self.armed {
609 let _ = self
610 .forwarding
611 .cancel_module_control_rpc(self.endpoint, self.corr);
612 }
613 }
614}
615
616impl RouteBindReservationGuard {
617 fn new(forwarding: Arc<ForwardingTable>, endpoint: ModuleEndpointId, relay_corr: u64) -> Self {
618 Self {
619 forwarding,
620 endpoint,
621 relay_corr,
622 armed: true,
623 }
624 }
625
626 fn release_and_disarm(&mut self) {
627 if !self.armed {
628 return;
629 }
630 if let Ok(Some(target)) = self.forwarding.abort_pending_relay(
631 self.endpoint,
632 self.relay_corr,
633 RouteBindRelayOutcome::ModuleGone("route.open handler canceled".to_string()),
634 ) {
635 send_goodbye_target_best_effort(
636 &self.forwarding.counters(),
637 &target,
638 "canceled route.bind",
639 );
640 }
641 self.armed = false;
642 }
643
644 fn disarm(&mut self) {
645 self.armed = false;
646 }
647}
648
649impl Drop for RouteBindReservationGuard {
650 fn drop(&mut self) {
651 self.release_and_disarm();
652 }
653}
654
655#[derive(Debug, Clone, Default)]
685pub(crate) struct RouteBindBreakers {
686 modules: Arc<Mutex<HashMap<String, ModuleBreakerState>>>,
687}
688
689#[derive(Debug, Clone, Default)]
690pub(crate) struct RouteBindConcurrency {
691 modules: Arc<Mutex<HashMap<String, usize>>>,
692}
693
694struct RouteBindConcurrencyGuard {
695 concurrency: RouteBindConcurrency,
696 module_id: String,
697}
698
699impl RouteBindConcurrency {
700 fn try_admit(&self, module_id: &str, limit: usize) -> Result<RouteBindConcurrencyGuard, usize> {
703 let mut modules = self
704 .modules
705 .lock()
706 .expect("route.bind concurrency mutex poisoned");
707 let in_flight = modules.entry(module_id.to_string()).or_default();
708 if *in_flight >= limit {
709 return Err(*in_flight);
710 }
711 *in_flight += 1;
712 Ok(RouteBindConcurrencyGuard {
713 concurrency: self.clone(),
714 module_id: module_id.to_string(),
715 })
716 }
717}
718
719impl Drop for RouteBindConcurrencyGuard {
720 fn drop(&mut self) {
721 let mut modules = self
722 .concurrency
723 .modules
724 .lock()
725 .expect("route.bind concurrency mutex poisoned");
726 let remove = {
727 let in_flight = modules
728 .get_mut(&self.module_id)
729 .expect("admitted route.bind has a concurrency entry");
730 *in_flight -= 1;
731 *in_flight == 0
732 };
733 if remove {
734 modules.remove(&self.module_id);
735 }
736 }
737}
738
739#[derive(Debug, Default)]
740struct ModuleBreakerState {
741 consecutive_timeouts: u32,
743 cooldown_until: Option<Instant>,
746 probe_in_flight: Option<Arc<()>>,
751}
752
753enum RouteBindAdmission<'a> {
755 Admitted {
756 guard: RouteBindBreakerGuard<'a>,
757 probe: bool,
760 },
761 Refused {
762 consecutive_timeouts: u32,
763 retry_in: Duration,
767 probe_in_flight: bool,
768 },
769}
770
771struct RouteBindBreakerGuard<'a> {
778 breakers: RouteBindBreakers,
779 module_id: &'a str,
780 probe_token: Option<Arc<()>>,
781 settled: bool,
782}
783
784impl RouteBindBreakerGuard<'_> {
785 fn record_accepted(&mut self) -> bool {
789 self.settled = true;
790 self.breakers.record_accepted(self.module_id)
791 }
792
793 fn record_timeout(&mut self, threshold: u32, cooldown: Duration) -> Option<BreakerOpened> {
796 self.settled = true;
797 self.breakers.record_timeout(
798 self.module_id,
799 self.probe_token.as_ref(),
800 threshold,
801 cooldown,
802 )
803 }
804
805 fn record_inconclusive(&mut self) {
814 self.settled = true;
815 self.breakers
816 .record_inconclusive(self.module_id, self.probe_token.as_ref());
817 }
818}
819
820impl Drop for RouteBindBreakerGuard<'_> {
821 fn drop(&mut self) {
822 if !self.settled {
823 self.breakers
824 .record_inconclusive(self.module_id, self.probe_token.as_ref());
825 }
826 }
827}
828
829struct BreakerOpened {
833 consecutive_timeouts: u32,
834 reopened_after_probe: bool,
837}
838
839impl RouteBindBreakers {
840 fn lock(&self) -> std::sync::MutexGuard<'_, HashMap<String, ModuleBreakerState>> {
841 self.modules
842 .lock()
843 .expect("route.bind breaker mutex poisoned")
844 }
845
846 fn admit<'a>(&self, module_id: &'a str) -> RouteBindAdmission<'a> {
849 let admitted = |probe_token: Option<Arc<()>>| RouteBindAdmission::Admitted {
850 probe: probe_token.is_some(),
851 guard: RouteBindBreakerGuard {
852 breakers: self.clone(),
853 module_id,
854 probe_token,
855 settled: false,
856 },
857 };
858
859 let mut modules = self.lock();
860 let Some(state) = modules.get_mut(module_id) else {
861 return admitted(None);
862 };
863 let Some(cooldown_until) = state.cooldown_until else {
864 return admitted(None);
865 };
866 if state.probe_in_flight.is_some() {
867 return RouteBindAdmission::Refused {
868 consecutive_timeouts: state.consecutive_timeouts,
869 retry_in: Duration::ZERO,
870 probe_in_flight: true,
871 };
872 }
873 let now = Instant::now();
874 if now < cooldown_until {
875 return RouteBindAdmission::Refused {
876 consecutive_timeouts: state.consecutive_timeouts,
877 retry_in: cooldown_until - now,
878 probe_in_flight: false,
879 };
880 }
881 let token = Arc::new(());
882 state.probe_in_flight = Some(Arc::clone(&token));
883 admitted(Some(token))
884 }
885
886 fn record_accepted(&self, module_id: &str) -> bool {
887 self.lock()
888 .remove(module_id)
889 .is_some_and(|state| state.cooldown_until.is_some())
890 }
891
892 fn record_timeout(
893 &self,
894 module_id: &str,
895 probe_token: Option<&Arc<()>>,
896 threshold: u32,
897 cooldown: Duration,
898 ) -> Option<BreakerOpened> {
899 let mut modules = self.lock();
900 let state = modules.entry(module_id.to_string()).or_default();
901 let was_open = state.cooldown_until.is_some();
902 let was_probe = Self::owns_probe(state, probe_token);
903 if was_probe {
904 state.probe_in_flight = None;
905 }
906 state.consecutive_timeouts = state.consecutive_timeouts.saturating_add(1);
907 if state.consecutive_timeouts < threshold {
908 return None;
909 }
910 state.cooldown_until = Some(Instant::now() + cooldown);
911 Some(BreakerOpened {
912 consecutive_timeouts: state.consecutive_timeouts,
913 reopened_after_probe: was_open && was_probe,
914 })
915 }
916
917 fn owns_probe(state: &ModuleBreakerState, token: Option<&Arc<()>>) -> bool {
918 state
922 .probe_in_flight
923 .as_ref()
924 .zip(token)
925 .is_some_and(|(active, token)| Arc::ptr_eq(active, token))
926 }
927
928 fn record_inconclusive(&self, module_id: &str, probe_token: Option<&Arc<()>>) {
929 if let Some(state) = self.lock().get_mut(module_id) {
930 if Self::owns_probe(state, probe_token) {
931 state.probe_in_flight = None;
932 }
933 }
934 }
935
936 pub(crate) fn reset_for_new_module_connection(&self, module_id: &str) -> Option<u32> {
951 self.lock()
952 .remove(module_id)
953 .map(|state| state.consecutive_timeouts)
954 .filter(|discarded| *discarded > 0)
955 }
956
957 fn open_snapshot(&self) -> Option<serde_json::Value> {
964 let now = Instant::now();
965 let modules = self.lock();
966 let open = modules
967 .iter()
968 .filter_map(|(module_id, state)| {
969 let cooldown_until = state.cooldown_until?;
970 Some((
971 module_id.clone(),
972 serde_json::json!({
973 "consecutive_timeouts": state.consecutive_timeouts,
974 "cooldown_remaining_ms":
975 cooldown_until.saturating_duration_since(now).as_millis() as u64,
976 "probe_in_flight": state.probe_in_flight.is_some(),
977 }),
978 ))
979 })
980 .collect::<serde_json::Map<String, serde_json::Value>>();
981 (!open.is_empty()).then_some(serde_json::Value::Object(open))
982 }
983}
984
985impl ControlHandler {
986 pub fn new(registry: Arc<Registry>) -> Self {
987 Self::with_forwarding(registry, Arc::new(ForwardingTable::default()))
988 }
989
990 pub fn with_forwarding(registry: Arc<Registry>, forwarding: Arc<ForwardingTable>) -> Self {
991 let counters = forwarding.counters();
992 let route_bind_breakers = forwarding.route_bind_breakers();
996 let route_bind_concurrency = forwarding.route_bind_concurrency();
997 let route_outages = forwarding.route_outages();
998 Self {
999 registry,
1000 activity: None,
1001 activity_streams: Arc::new(Mutex::new(HashMap::new())),
1002 forwarding,
1003 process_liveness: None,
1004 supervisor: SupervisorHandle::new(),
1005 resource_tracker: Arc::new(crate::resource_totals::Tracker::new(
1006 env!("CARGO_PKG_VERSION").to_owned(),
1007 )),
1008 agent_declarations: agent_commands::Queues::default(),
1009 list_admission: Arc::new(tokio::sync::Semaphore::new(4)),
1010 subc_capabilities: Arc::from([
1011 CAP_MANIFEST_REGISTRATION.to_string(),
1012 CAP_CHANNEL_LIFECYCLE.to_string(),
1013 CAP_PING_PONG.to_string(),
1014 CAP_SESSION_ATTACH.to_string(),
1015 CAP_ADMISSION_FACTS_RELAY.to_string(),
1016 CAP_SCOPES_V1.to_string(),
1017 subc_protocol::scope::SCOPE_FEDERATED_CAPABILITY.to_string(),
1018 CAP_ROUTE_ROLE_VERSIONS_V1.to_string(),
1019 subc_protocol::manifest::CAP_CATALOG_EVENTS_V1.to_string(),
1020 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1.to_string(),
1021 subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2.to_string(),
1022 subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1.to_string(),
1023 subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY.to_string(),
1024 subc_protocol::scope::WORKER_RUNS_CAPABILITY.to_string(),
1025 subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY.to_string(),
1026 subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY.to_string(),
1027 subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY.to_string(),
1028 subc_control::ACTIVITY_LEDGER_CAPABILITY.to_string(),
1029 ]),
1030 route_bind_relay_timeout: DEFAULT_ROUTE_BIND_RELAY_TIMEOUT,
1031 route_bind_relay_timeouts: BTreeMap::new(),
1032 route_bind_breakers,
1033 route_bind_concurrency,
1034 route_outages,
1035 route_bind_breaker_threshold: DEFAULT_ROUTE_BIND_BREAKER_THRESHOLD,
1036 route_bind_breaker_cooldown: DEFAULT_ROUTE_BIND_BREAKER_COOLDOWN,
1037 health_probe_timeout: DEFAULT_HEALTH_PROBE_TIMEOUT,
1038 storage_config: None,
1039 machine_id: None,
1040 admission_facts_carrier_module_id: None,
1041 admission_facts_targets: None,
1042 scopes: Arc::new(RwLock::new(ScopeTable::new(
1043 crate::daemon_config::default_scope_authority_owners(),
1044 ))),
1045 scope_deadline_added: Arc::new(tokio::sync::Notify::new()),
1046 scope_authority_owners: crate::daemon_config::default_scope_authority_owners(),
1047 federated_scope_owner: crate::daemon_config::default_federated_scope_owner(),
1048 privileged_operator_prompt: PrivilegedOperatorPrompt::default(),
1049 caller_endpoints: Arc::new(Mutex::new(HashMap::new())),
1050 build_class_warned: Arc::new(Mutex::new(HashSet::new())),
1051 observed_peers: Arc::new(Mutex::new(HashMap::new())),
1052 local_connections: Arc::new(Mutex::new(HashSet::new())),
1053 operator_app_policy: Default::default(),
1054 operator_apps: Arc::new(Mutex::new(HashMap::new())),
1055 launch_table: std::sync::Arc::new(crate::app_open::LaunchTable::new()),
1056 app_open_launch_lock: std::sync::Arc::new(tokio::sync::Mutex::new(())),
1057 #[cfg(test)]
1058 app_open_test_platform: None,
1059 #[cfg(test)]
1060 app_open_test_launch: None,
1061 #[cfg(windows)]
1062 windows_verify: None,
1063 #[cfg(windows)]
1064 windows_flash: None,
1065 #[cfg(windows)]
1066 windows_track: None,
1067 #[cfg(all(test, windows))]
1068 windows_spawn_attempts: Arc::new(std::sync::atomic::AtomicU64::new(0)),
1069 #[cfg(test)]
1070 caller_lookup: None,
1071 hello_launch_nonces: Arc::new(Mutex::new(HelloLaunchNonces::default())),
1072 rescan: None,
1073 connected_clients: ConnectedClients::new(),
1074 local_endpoint_unavailable: None,
1075 counters,
1076 capability_evaluator: Arc::new(CapabilityRequirementEvaluator::new()),
1077 daemon_shutdown: DaemonShutdownFlag::default(),
1078 daemon_provenance: DaemonProvenanceFacts::default(),
1079 #[cfg(test)]
1080 control_dispatch_delay: None,
1081 #[cfg(test)]
1082 provenance_probe_override: None,
1083 #[cfg(test)]
1084 route_bind_relay_sent_hold: None,
1085 #[cfg(test)]
1086 hello_registry_to_forwarding_hold: None,
1087 }
1088 }
1089
1090 pub fn with_storage_config(
1093 mut self,
1094 storage_config: Option<crate::daemon_config::StorageConfig>,
1095 ) -> Self {
1096 self.storage_config = storage_config;
1097 self
1098 }
1099
1100 pub fn with_activity_ledger(mut self, run_dir: &Path, incarnation: String) -> Self {
1103 self.activity = Some(crate::activity::ActivityLedger::open(
1104 run_dir.join("activity.jsonl"),
1105 incarnation,
1106 ));
1107 self.supervisor.set_activity(self.activity.clone());
1108 self
1109 }
1110
1111 pub fn with_machine_id(mut self, machine_id: Option<crate::machine_id::MachineId>) -> Self {
1114 self.machine_id = machine_id;
1115 self
1116 }
1117
1118 pub fn with_admission_facts_config(
1122 mut self,
1123 carrier_module_id: Option<String>,
1124 targets: Option<Vec<String>>,
1125 ) -> Self {
1126 self.admission_facts_carrier_module_id = carrier_module_id;
1127 self.admission_facts_targets = targets;
1128 self
1129 }
1130
1131 pub fn with_scope_authority_owners(mut self, owners: Vec<String>) -> Self {
1135 self.scopes = Arc::new(RwLock::new(
1136 ScopeTable::new(owners.iter().cloned())
1137 .with_federated_owner(self.federated_scope_owner.clone()),
1138 ));
1139 self.scope_authority_owners = owners;
1140 self
1141 }
1142
1143 pub fn with_federated_scope_owner(mut self, owner: String) -> Self {
1145 self.scopes = Arc::new(RwLock::new(
1146 ScopeTable::new(self.scope_authority_owners.iter().cloned())
1147 .with_federated_owner(owner.clone()),
1148 ));
1149 self.federated_scope_owner = owner;
1150 self
1151 }
1152
1153 pub fn with_privileged_operator_prompt(mut self, mode: PrivilegedOperatorPrompt) -> Self {
1155 self.privileged_operator_prompt = mode;
1156 self
1157 }
1158
1159 pub fn with_route_bind_relay_timeout(mut self, timeout: Duration) -> Self {
1162 self.route_bind_relay_timeout = timeout;
1163 self
1164 }
1165
1166 pub fn with_route_bind_relay_timeouts(
1172 mut self,
1173 timeouts: impl IntoIterator<Item = (String, Duration)>,
1174 ) -> Self {
1175 self.route_bind_relay_timeouts = timeouts.into_iter().collect();
1176 self
1177 }
1178
1179 pub fn route_bind_relay_timeout_for(&self, module_id: &str) -> Duration {
1185 self.route_bind_relay_timeouts
1186 .get(module_id)
1187 .copied()
1188 .unwrap_or(self.route_bind_relay_timeout)
1189 }
1190
1191 pub fn with_route_bind_breaker(mut self, threshold: u32, cooldown: Duration) -> Self {
1199 self.route_bind_breaker_threshold = threshold.max(1);
1200 self.route_bind_breaker_cooldown = cooldown;
1201 self
1202 }
1203
1204 #[cfg(test)]
1205 pub(crate) fn with_health_probe_timeout(mut self, timeout: Duration) -> Self {
1206 self.health_probe_timeout = timeout;
1207 self
1208 }
1209
1210 #[cfg(test)]
1211 pub(crate) fn with_control_dispatch_delay(mut self, delay: Duration) -> Self {
1212 self.control_dispatch_delay = Some(delay);
1213 self
1214 }
1215
1216 #[cfg(test)]
1220 pub(crate) fn with_route_bind_relay_sent_hold(
1221 mut self,
1222 hold: Arc<tokio::sync::Barrier>,
1223 ) -> Self {
1224 self.route_bind_relay_sent_hold = Some(hold);
1225 self
1226 }
1227
1228 #[cfg(test)]
1233 pub(crate) fn with_hello_registry_to_forwarding_hold(
1234 mut self,
1235 hold: Arc<std::sync::Barrier>,
1236 ) -> Self {
1237 self.hello_registry_to_forwarding_hold = Some(hold);
1238 self
1239 }
1240
1241 #[cfg(test)]
1242 fn hold_between_registry_and_forwarding(&self) {
1243 if let Some(hold) = &self.hello_registry_to_forwarding_hold {
1244 hold.wait();
1245 hold.wait();
1246 }
1247 }
1248
1249 pub fn with_process_liveness(
1250 mut self,
1251 process_liveness: Arc<dyn ModuleProcessLiveness>,
1252 ) -> Self {
1253 self.process_liveness = Some(process_liveness);
1254 self
1255 }
1256
1257 pub fn with_supervisor(mut self, supervisor: SupervisorHandle) -> Self {
1258 self.supervisor = supervisor;
1259 self.supervisor.set_activity(self.activity.clone());
1260 self
1261 }
1262
1263 pub fn with_daemon_provenance(
1264 mut self,
1265 pid: u32,
1266 started_at_ms: u64,
1267 executable_path: Option<PathBuf>,
1268 build_git_sha: Option<String>,
1269 build_lock_digest: Option<String>,
1270 ) -> Self {
1271 let executable_identity = executable_path.as_deref().and_then(spawned_file_identity);
1272 let process_start_time = process_start_time(pid);
1273 self.daemon_provenance = DaemonProvenanceFacts {
1274 build: DaemonBuildProvenance {
1275 build_git_sha,
1276 build_lock_digest,
1277 },
1278 pid: Some(pid),
1279 started_at_ms: Some(started_at_ms),
1280 start_clock: None,
1281 executable_path,
1282 executable_identity,
1283 process_start_time,
1284 probe: ExecutableIdentityProbe::default(),
1285 };
1286 self
1287 }
1288
1289 pub(crate) fn with_daemon_start_clock(mut self, clock: crate::clock::StartClock) -> Self {
1290 self.daemon_provenance.start_clock = Some(clock);
1291 self
1292 }
1293
1294 pub fn with_wall_clock(self, clock: impl Fn() -> u64 + Send + Sync + 'static) -> Self {
1296 self.scopes
1297 .write()
1298 .unwrap_or_else(|poisoned| poisoned.into_inner())
1299 .set_wall_clock(Arc::new(clock));
1300 self
1301 }
1302
1303 #[cfg(test)]
1304 fn with_provenance_probe_result(mut self, result: subc_control::RunningImageAgreement) -> Self {
1305 self.provenance_probe_override = Some(result);
1306 self
1307 }
1308
1309 pub fn with_capability_config(
1313 self,
1314 modules: impl IntoIterator<Item = (String, bool)>,
1315 reserved_capabilities: BTreeMap<String, String>,
1316 ) -> Self {
1317 self.capability_evaluator
1318 .configure(modules, reserved_capabilities);
1319 self
1320 }
1321
1322 pub(crate) fn with_daemon_shutdown_flag(mut self, flag: DaemonShutdownFlag) -> Self {
1326 self.daemon_shutdown = flag;
1327 self
1328 }
1329
1330 pub fn with_supervisor_rescan(
1331 mut self,
1332 supervisor: Supervisor,
1333 config_path: impl Into<PathBuf>,
1334 configured_port: Option<u16>,
1335 ) -> Self {
1336 self.rescan = Some(SupervisorRescanContext {
1337 supervisor,
1338 config_path: config_path.into(),
1339 configured_port,
1340 storage_config: self.storage_config.clone(),
1341 admission_facts_carrier_module_id: self.admission_facts_carrier_module_id.clone(),
1342 admission_facts_targets: self.admission_facts_targets.clone(),
1343 scope_authority_owners: self.scope_authority_owners.clone(),
1344 federated_scope_owner: self.federated_scope_owner.clone(),
1345 privileged_operator_prompt: self.privileged_operator_prompt,
1346 generation: Arc::default(),
1347 });
1348 self
1349 }
1350
1351 pub fn with_connected_clients(mut self, connected_clients: ConnectedClients) -> Self {
1352 self.connected_clients = connected_clients;
1353 self
1354 }
1355
1356 pub fn with_local_endpoint_unavailable(mut self, reason: Option<String>) -> Self {
1359 self.local_endpoint_unavailable = reason;
1360 self
1361 }
1362
1363 pub fn forwarding(&self) -> Arc<ForwardingTable> {
1364 Arc::clone(&self.forwarding)
1365 }
1366
1367 pub(crate) fn counters(&self) -> DaemonCounters {
1368 self.counters.clone()
1369 }
1370
1371 pub fn spawn_capability_deadline_loop(self: Arc<Self>) {
1374 tokio::spawn(async move {
1375 loop {
1376 self.capability_evaluator
1377 .wait_for_change_or_deadline()
1378 .await;
1379 self.refresh_capability_requirements();
1380 }
1381 });
1382 }
1383
1384 pub(crate) fn spawn_scope_expiry_loop(self: Arc<Self>) {
1390 tokio::spawn(async move {
1391 loop {
1392 let has_deadlines = self
1393 .scopes
1394 .read()
1395 .unwrap_or_else(|poisoned| poisoned.into_inner())
1396 .has_deadlines();
1397 if !has_deadlines {
1398 self.scope_deadline_added.notified().await;
1399 continue;
1400 }
1401 tokio::time::sleep(Duration::from_secs(1)).await;
1402 if let Err(error) = self.sweep_expired_scopes() {
1403 warn!(%error, "scope expiry sweep failed");
1404 }
1405 }
1406 });
1407 }
1408
1409 pub(crate) fn sweep_expired_scopes(&self) -> Result<(), RouterError> {
1410 let mut table = self
1411 .scopes
1412 .write()
1413 .unwrap_or_else(|poisoned| poisoned.into_inner());
1414 let swept = table.sweep_expired();
1415 let drained = self.publish_scope_changes(&swept.tag_changes, &swept.expired)?;
1416 drop(table);
1417 self.close_scope_drained_routes(drained);
1418 Ok(())
1419 }
1420
1421 fn publish_scope_changes(
1422 &self,
1423 changes: &[crate::scopes::ScopeTagChange],
1424 expired: &[crate::scopes::ScopeExpired],
1425 ) -> Result<Vec<crate::forwarding::ScopeDrainedRoute>, RouterError> {
1426 if changes.is_empty() {
1427 return Ok(Vec::new());
1428 }
1429 let drained = self
1430 .forwarding
1431 .publish_scope_changes(changes)
1432 .map_err(RouterError::Forwarding)?;
1433 let mut counts = HashMap::new();
1434 for route in &drained {
1435 *counts
1436 .entry((
1437 route.scope.owner.as_str(),
1438 route.scope.scope_ref.as_str(),
1439 route.scope.tag.scope_epoch,
1440 ))
1441 .or_insert(0usize) += 1;
1442 }
1443 for scope in expired {
1444 let routes_closed = counts
1445 .get(&(
1446 scope.owner.as_str(),
1447 scope.scope_ref.as_str(),
1448 scope.scope_epoch,
1449 ))
1450 .copied()
1451 .unwrap_or(0);
1452 warn!(owner = %scope.owner, scope_ref = %scope.scope_ref,
1453 scope_epoch = scope.scope_epoch, expires_at_ms = scope.expires_at_ms,
1454 routes_closed, "scope expired");
1455 }
1456 Ok(drained)
1457 }
1458
1459 fn runtime_capability_snapshot(
1460 &self,
1461 ) -> Result<(Vec<RuntimeModule>, Vec<RegisteredModule>), RouterError> {
1462 let runtime = self
1463 .supervisor
1464 .list()
1465 .into_iter()
1466 .map(|module| {
1467 let status = module.status().map_err(|err| {
1468 RouterError::backend(0, 0, format!("failed to read capability status: {err}"))
1469 })?;
1470 Ok(RuntimeModule {
1471 module_id: status.module_id,
1472 state: status.state,
1473 enabled: status.enabled,
1474 })
1475 })
1476 .collect::<Result<Vec<_>, RouterError>>()?;
1477 let registrations = self.registry.list_registered_modules().map_err(|err| {
1483 RouterError::backend(
1484 0,
1485 0,
1486 format!("failed to list capability registrations: {err}"),
1487 )
1488 })?;
1489 let registrations = registrations
1490 .into_iter()
1491 .map(|registration| RegisteredModule {
1492 module_id: registration.manifest.module_id,
1493 module_version: registration.manifest.module_version,
1494 capabilities: registration.manifest.capabilities,
1495 })
1496 .collect();
1497 Ok((runtime, registrations))
1498 }
1499
1500 fn apply_registration_capabilities(&self, registration: &crate::registry::ModuleRegistration) {
1507 let cached_registration = RegisteredModule {
1508 module_id: registration.manifest.module_id.clone(),
1509 module_version: registration.manifest.module_version.clone(),
1510 capabilities: registration.manifest.capabilities.clone(),
1511 };
1512 let change = self.capability_evaluator.record_hello(&cached_registration);
1513 log_manifest_change(&cached_registration.module_id, &change);
1514 if capability_census_trigger(None, registration.manifest.capabilities.as_ref()) {
1515 self.enforce_capability_denies();
1516 }
1517 self.refresh_capability_requirements();
1518 }
1519
1520 pub(crate) fn install_swap_promotion_observer(self: &Arc<Self>) {
1524 let observer: std::sync::Weak<dyn crate::supervise::SwapPromotionObserver> =
1525 Arc::downgrade(self) as std::sync::Weak<ControlHandler>;
1526 self.supervisor.set_swap_promotion_observer(observer);
1527 }
1528
1529 pub fn refresh_capability_requirements(&self) {
1530 match self.runtime_capability_snapshot() {
1531 Ok((runtime, registrations)) => {
1532 log_requirement_events(
1533 self.capability_evaluator
1534 .evaluate_now(&runtime, ®istrations),
1535 &self.daemon_shutdown,
1536 );
1537 }
1538 Err(err) => warn!(error = %err, "failed to recompute capability requirements"),
1539 }
1540 }
1541
1542 fn enforce_capability_denies(&self) {
1546 let registrations = match self.registry.list_registered_modules() {
1547 Ok(snapshot) => snapshot,
1548 Err(err) => {
1549 warn!(error = %err, "failed to read registrations for capability deny census");
1550 return;
1551 }
1552 };
1553 let manifests = registrations
1554 .into_iter()
1555 .map(|registration| {
1556 (
1557 registration.manifest.module_id.clone(),
1558 registration.manifest,
1559 )
1560 })
1561 .collect::<BTreeMap<_, _>>();
1562 let census = match self.forwarding.route_census(None) {
1563 Ok(census) => census,
1564 Err(err) => {
1565 warn!(error = %err, "failed to read route census for capability deny enforcement");
1566 return;
1567 }
1568 };
1569
1570 for (target_module_id, routes) in census {
1571 let Some(target_manifest) = manifests.get(&target_module_id) else {
1572 continue;
1573 };
1574 let mut closed_routes = Vec::new();
1575 let mut goodbyes = Vec::new();
1576 for route in routes {
1577 let Principal::Reserved {
1578 module_id: opening_module_id,
1579 } = &route.principal
1580 else {
1581 continue;
1582 };
1583 let Some(opening_manifest) = manifests.get(opening_module_id) else {
1584 continue;
1585 };
1586 let Some(capability) = denied_capability(opening_manifest, target_manifest) else {
1587 continue;
1588 };
1589
1590 match self.forwarding.release_client_route(
1591 route.goodbye_target.connection_id,
1592 route.goodbye_target.channel,
1593 route.goodbye_target.epoch,
1594 ) {
1595 Ok(RouteRelease::Removed(module_goodbye)) => {
1596 warn!(
1597 opening_module_id,
1598 target_module_id,
1599 capability,
1600 "force-closing route because an attested capability deny edge now matches"
1601 );
1602 goodbyes.push(module_goodbye);
1603 goodbyes.push(route.goodbye_target.clone());
1604 closed_routes.push(route);
1605 }
1606 Ok(RouteRelease::Stale | RouteRelease::Absent) => {}
1607 Err(err) => warn!(
1608 opening_module_id,
1609 target_module_id,
1610 capability,
1611 error = %err,
1612 "failed to force-close capability-denied route"
1613 ),
1614 }
1615 }
1616
1617 if closed_routes.is_empty() {
1618 continue;
1619 }
1620 send_route_control_pushes(
1621 &self.forwarding,
1622 closed_routes,
1623 ClientControlPush::RouteClosed {
1624 module_id: target_module_id,
1625 channels: Vec::new(),
1626 reason: RouteCloseReason::CapabilityDenied,
1627 drained: false,
1628 abandoned: 0,
1629 excluded_subscriptions: 0,
1630 terminal: Some(false),
1631 },
1632 );
1633 self.emit_route_goodbyes(goodbyes);
1636 }
1637 }
1638
1639 fn not_ready_reason(
1645 &self,
1646 registration: &crate::registry::ModuleRegistration,
1647 ) -> Option<NotReadyReason> {
1648 if !registration.ready {
1649 return Some(NotReadyReason {
1650 reason: NotReadyReason::DECLARED_NOT_READY.to_string(),
1651 capability: None,
1652 });
1653 }
1654 self.first_unprovided_required_capability(registration)
1655 .map(|capability| NotReadyReason {
1656 reason: NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED.to_string(),
1657 capability: Some(capability),
1658 })
1659 }
1660
1661 fn first_unprovided_required_capability(
1679 &self,
1680 registration: &crate::registry::ModuleRegistration,
1681 ) -> Option<String> {
1682 let required = registration
1683 .manifest
1684 .capabilities
1685 .iter()
1686 .flat_map(|declarations| declarations.requires.iter())
1687 .filter(|requirement| requirement.need == CapabilityNeed::Required)
1688 .map(|requirement| requirement.capability.as_str())
1689 .collect::<BTreeSet<_>>();
1690 if required.is_empty() {
1691 return None;
1692 }
1693 let module_id = registration.manifest.module_id.as_str();
1694 let verdict = |capability: &str| self.capability_evaluator.verdict(module_id, capability);
1695 if required
1696 .iter()
1697 .any(|capability| verdict(capability).is_none())
1698 {
1699 self.refresh_capability_requirements();
1700 }
1701 required
1702 .into_iter()
1703 .find(|capability| verdict(capability) != Some(CapabilityVerdict::Provided))
1704 .map(str::to_string)
1705 }
1706
1707 fn capability_requirement_statuses(&self) -> Vec<CapabilityRequirementStatus> {
1708 self.capability_evaluator
1709 .statuses()
1710 .into_iter()
1711 .map(capability_requirement_status)
1712 .collect()
1713 }
1714
1715 fn deregister_connection(
1722 &self,
1723 connection_id: ConnectionId,
1724 reason: RegistrationEndReason,
1725 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
1726 self.registry
1727 .deregister_connection_with_reason(connection_id, reason)
1728 }
1729
1730 pub(crate) fn route_open_target(&self, frame: &Frame) -> Option<String> {
1731 if frame.header.channel != 0 || frame.header.ty != FrameType::Request {
1732 return None;
1733 }
1734 let Ok(ClientControlRequest::RouteOpen { target, .. }) =
1735 parse_client_control_request(&frame.body)
1736 else {
1737 return None;
1738 };
1739 Some(target_module_id(&target).to_string())
1740 }
1741
1742 pub(crate) fn route_open_capacity_refusal(
1743 &self,
1744 ctx: &RouteCtx,
1745 frame: &Frame,
1746 target_module_id: &str,
1747 in_flight: usize,
1748 limit: usize,
1749 ) -> Result<Frame, RouterError> {
1750 let caller = self.route_open_caller_from_frame(frame);
1753 self.route_open_admission_refusal_frame(
1754 ctx,
1755 &caller,
1756 frame,
1757 target_module_id,
1758 "open_admission_full",
1759 (in_flight, limit),
1760 format!(
1761 "connection already has {in_flight} route.open binds in flight (limit {limit}); retry after one settles"
1762 ),
1763 )
1764 }
1765
1766 fn route_open_target_capacity_refusal(
1767 &self,
1768 ctx: &RouteCtx,
1769 caller: &RouteOpenCaller,
1770 frame: &Frame,
1771 target_module_id: &str,
1772 in_flight: usize,
1773 ) -> Result<Frame, RouterError> {
1774 self.route_open_admission_refusal_frame(
1775 ctx,
1776 caller,
1777 frame,
1778 target_module_id,
1779 "target_binds_full",
1780 (in_flight, MAX_PENDING_ROUTE_BINDS_PER_TARGET),
1781 format!(
1782 "module_id '{target_module_id}' already has {in_flight} route.bind relays in flight; retry after one settles"
1783 ),
1784 )
1785 }
1786
1787 #[allow(clippy::too_many_arguments)]
1795 fn route_open_admission_refusal_frame(
1796 &self,
1797 ctx: &RouteCtx,
1798 caller: &RouteOpenCaller,
1799 frame: &Frame,
1800 target_module_id: &str,
1801 reason: &'static str,
1802 (in_flight, limit): (usize, usize),
1803 message: impl Into<String>,
1804 ) -> Result<Frame, RouterError> {
1805 let code = error_codes::TARGET_UNAVAILABLE;
1806 self.counters.increment_route_open_refused(code);
1807 info!(
1808 target: "control",
1809 code,
1810 reason,
1811 module_id = ?target_module_id,
1812 connection_id = ctx.connection_id.get(),
1813 principal = %caller.principal,
1814 scope = %caller.scope,
1815 harness = %caller.harness,
1816 in_flight,
1817 limit,
1818 "route.open refused"
1819 );
1820 control_error_frame(frame, code, message.into())
1821 }
1822
1823 #[cfg(test)]
1829 pub fn handle_control(
1830 &self,
1831 connection_id: ConnectionId,
1832 frame: Frame,
1833 ) -> Result<Vec<Frame>, RouterError> {
1834 match frame.header.ty {
1835 FrameType::Ping => Ok(vec![pong(&frame)?]),
1836 FrameType::Hello => self.handle_hello(connection_id, None, frame),
1837 FrameType::Goodbye => self.handle_goodbye(connection_id),
1838 ty => Ok(vec![control_error_frame(
1839 &frame,
1840 "unsupported_control_frame",
1841 format!("unsupported channel-0 frame {ty:?}"),
1842 )?]),
1843 }
1844 }
1845
1846 pub async fn handle_control_frame(
1847 &self,
1848 ctx: &RouteCtx,
1849 frame: Frame,
1850 ) -> Result<Vec<Frame>, RouterError> {
1851 self.handle_control_frame_timed(ctx, frame, None).await
1852 }
1853
1854 #[cfg(any(test, feature = "test-support"))]
1855 pub fn with_test_operator_app_policy(
1856 mut self,
1857 policy: crate::operator_apps::TestOperatorAppPolicy,
1858 ) -> Self {
1859 self.operator_app_policy = policy.into();
1860 self
1861 }
1862
1863 #[cfg(all(windows, any(test, feature = "test-support")))]
1867 pub fn with_test_windows_verifier(mut self, verify: crate::app_open::WindowsVerifyFn) -> Self {
1868 self.windows_verify = Some(verify);
1869 self
1870 }
1871
1872 #[cfg(all(windows, any(test, feature = "test-support")))]
1874 pub fn with_test_windows_flash(mut self, flash: crate::app_open::WindowsFlashFn) -> Self {
1875 self.windows_flash = Some(flash);
1876 self
1877 }
1878
1879 #[cfg(all(windows, any(test, feature = "test-support")))]
1883 pub fn with_test_windows_track(mut self, track: crate::app_open::WindowsTrackFn) -> Self {
1884 self.windows_track = Some(track);
1885 self
1886 }
1887
1888 pub(crate) fn record_captured_peer(
1889 &self,
1890 connection: ConnectionId,
1891 peer: subc_os::local_peer::CapturedPeer,
1892 ) {
1893 self.record_observed_peer(connection, peer.observation().clone());
1894 #[cfg(windows)]
1895 if let Some(stamp) = self.stamp_windows_launch(&peer) {
1896 self.operator_apps
1897 .lock()
1898 .unwrap_or_else(|p| p.into_inner())
1899 .insert(connection, stamp);
1900 }
1901 if let Some(stamp) = self.operator_app_policy.stamp(peer) {
1902 self.operator_apps
1903 .lock()
1904 .unwrap_or_else(|p| p.into_inner())
1905 .insert(connection, stamp);
1906 }
1907 }
1908
1909 pub(crate) fn record_local_connection(&self, connection: ConnectionId) {
1910 self.local_connections
1911 .lock()
1912 .unwrap_or_else(|p| p.into_inner())
1913 .insert(connection);
1914 }
1915
1916 fn connection_operator_app(
1919 &self,
1920 stamp: Option<&crate::operator_apps::OperatorAppStamp>,
1921 ) -> Option<subc_protocol::peer::OperatorApp> {
1922 stamp
1923 .filter(|stamp| stamp.delivery_enabled(&self.operator_app_policy))
1924 .map(|stamp| stamp.app.clone())
1925 }
1926
1927 fn handle_connection_describe(
1928 &self,
1929 ctx: &RouteCtx,
1930 frame: Frame,
1931 ) -> Result<Vec<Frame>, RouterError> {
1932 if serde_json::from_slice::<subc_control::ConnectionDescribeRequest>(&frame.body).is_err() {
1933 return Ok(vec![control_error_frame(
1934 &frame,
1935 "invalid_control_body",
1936 "malformed connection.describe body",
1937 )?]);
1938 }
1939 let local = self
1940 .local_connections
1941 .lock()
1942 .unwrap_or_else(|p| p.into_inner())
1943 .contains(&ctx.connection_id);
1944 let stamps = self.operator_apps.lock().unwrap_or_else(|p| p.into_inner());
1945 let operator_app = self.connection_operator_app(stamps.get(&ctx.connection_id));
1946 Ok(vec![control_response_body_frame(
1947 &frame,
1948 &subc_control::ConnectionDescribeReply::new(local, operator_app),
1949 ops::CONNECTION_DESCRIBE,
1950 )?])
1951 }
1952
1953 pub(crate) fn record_observed_peer(
1954 &self,
1955 connection: ConnectionId,
1956 peer: subc_os::local_peer::ObservedPeer,
1957 ) {
1958 use subc_os::local_peer::{ObservedPeer as OsPeer, PeerUser};
1959 let peer = match peer {
1960 OsPeer::Unobserved => ObservedPeer::unobserved(),
1961 OsPeer::Observed(peer) => {
1962 let observed = match peer.user {
1963 PeerUser::Uid(uid) => ObservedPeer::observed_uid(uid),
1964 PeerUser::Sid(sid) => ObservedPeer::observed_sid(sid),
1965 };
1966 observed
1967 .with_exe(
1968 peer.exe
1969 .and_then(|path| path.into_os_string().into_string().ok()),
1970 )
1971 .with_signing_id(peer.signing_id)
1972 }
1973 };
1974 self.observed_peers
1975 .lock()
1976 .unwrap_or_else(|p| p.into_inner())
1977 .insert(connection, peer);
1978 }
1979
1980 pub(crate) fn record_caller_endpoints(
1981 &self,
1982 connection: ConnectionId,
1983 endpoints: Option<(SocketAddr, SocketAddr)>,
1984 ) {
1985 let mut callers = self
1986 .caller_endpoints
1987 .lock()
1988 .unwrap_or_else(|p| p.into_inner());
1989 callers.remove(&connection);
1990 if let Some(endpoints) = endpoints {
1991 callers.insert(connection, endpoints);
1992 }
1993 }
1994
1995 #[cfg(test)]
1996 pub(crate) fn with_caller_lookup_for_test(
1997 mut self,
1998 lookup: impl Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync + 'static,
1999 ) -> Self {
2000 self.caller_lookup = Some(Arc::new(lookup));
2001 self
2002 }
2003
2004 async fn resolve_caller(&self, connection: ConnectionId) -> CallerIdentity {
2005 let endpoints = self
2006 .caller_endpoints
2007 .lock()
2008 .unwrap_or_else(|p| p.into_inner())
2009 .get(&connection)
2010 .copied();
2011 #[cfg(test)]
2012 let lookup = self.caller_lookup.clone();
2013 tokio::task::spawn_blocking(move || {
2014 #[cfg(test)]
2015 if let Some(lookup) = lookup {
2016 return lookup(endpoints);
2017 }
2018 endpoints
2019 .map(|(local, peer)| CallerIdentity::lookup_endpoints(local, peer))
2020 .unwrap_or_default()
2021 })
2022 .await
2023 .unwrap_or_default()
2024 }
2025
2026 pub(crate) fn defers_privileged_control(&self, ctx: &RouteCtx, frame: &Frame) -> bool {
2027 frame.header.channel == 0
2028 && frame.header.ty == FrameType::Request
2029 && self
2030 .forwarding
2031 .module_endpoint_for_connection(ctx.connection_id)
2032 .is_ok_and(|endpoint| endpoint.is_none())
2033 && parse_client_control_request(&frame.body)
2034 .ok()
2035 .is_some_and(|request| privileged_operation(&request).is_some())
2036 }
2037
2038 pub(crate) async fn handle_control_frame_timed(
2039 &self,
2040 ctx: &RouteCtx,
2041 frame: Frame,
2042 dispatch_started_at: Option<StdInstant>,
2043 ) -> Result<Vec<Frame>, RouterError> {
2044 match frame.header.ty {
2045 FrameType::Ping => Ok(vec![pong(&frame)?]),
2046 FrameType::Hello => {
2047 self.handle_hello(ctx.connection_id, Some(ctx.egress.clone()), frame)
2048 }
2049 FrameType::Goodbye => self.handle_goodbye(ctx.connection_id),
2050 FrameType::Cancel => {
2051 if self
2054 .agent_declarations
2055 .cancel(ctx.connection_id, frame.header.corr)
2056 || self
2057 .forwarding
2058 .operator_confirms()
2059 .cancel(ctx.connection_id, frame.header.corr)
2060 || self
2061 .supervisor
2062 .cancel_spawn_subscription(ctx.connection_id, frame.header.corr)
2063 || self.cancel_activity_subscription(ctx, &frame)?
2064 {
2065 Ok(Vec::new())
2066 } else {
2067 Ok(vec![control_error_frame(
2068 &frame,
2069 "unknown_subscription",
2070 "no subscription has this correlation id",
2071 )?])
2072 }
2073 }
2074 FrameType::Request => {
2075 let op = serde_json::from_slice::<ControlOpProbe>(&frame.body).ok();
2078 if op
2079 .as_ref()
2080 .is_some_and(|probe| probe.op == ops::ACTIVITY_RECORD)
2081 {
2082 return self.handle_activity_record(ctx, frame).await;
2083 }
2084 if op.as_ref().is_some_and(|probe| {
2085 probe.op == ops::ACTIVITY_QUERY || probe.op == ops::ACTIVITY_SUBSCRIBE
2086 }) {
2087 return self.handle_activity_read(ctx, frame).await;
2088 }
2089 if op
2090 .as_ref()
2091 .is_some_and(|probe| probe.op == "operator.confirm")
2092 {
2093 return self.handle_operator_confirm(ctx, frame);
2094 }
2095 if op.as_ref().is_some_and(|probe| probe.op == ops::APP_OPEN) {
2096 return self
2097 .observe_control_wait(
2098 ctx,
2099 None,
2100 ops::APP_OPEN,
2101 self.handle_app_open(ctx, frame),
2102 )
2103 .await;
2104 }
2105 if op
2106 .as_ref()
2107 .is_some_and(|probe| probe.op == ops::CONNECTION_DESCRIBE)
2108 {
2109 return self.handle_connection_describe(ctx, frame);
2110 }
2111 if op
2112 .as_ref()
2113 .is_some_and(|probe| probe.op == subc_control::SCOPE_SERVED_BY_OP)
2114 {
2115 return self.handle_scope_served_by(ctx.connection_id, frame);
2116 }
2117 if self
2118 .forwarding
2119 .module_endpoint_for_connection(ctx.connection_id)
2120 .map_err(RouterError::Forwarding)?
2121 .is_some()
2122 {
2123 if !is_known_module_request_op(&frame.body) {
2124 return Ok(vec![control_error_frame(
2125 &frame,
2126 "unsupported_control_frame",
2127 "module-originated channel-0 REQUEST is not supported",
2128 )?]);
2129 }
2130 let request = match parse_module_control_request_from_module(&frame.body) {
2131 Ok(request) => request,
2132 Err((err, ControlRequestBodyError::UnknownOp)) => {
2133 return Ok(vec![control_error_frame(
2134 &frame,
2135 "unsupported_control_frame",
2136 format!("unsupported module-originated channel-0 REQUEST: {err}"),
2137 )?])
2138 }
2139 Err((err, ControlRequestBodyError::InvalidBody)) => {
2140 return Ok(vec![control_error_frame(
2141 &frame,
2142 "invalid_control_body",
2143 format!("malformed module control body: {err}"),
2144 )?])
2145 }
2146 };
2147 let op = module_control_request_op(&request);
2148 let corr = frame.header.corr;
2149 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2150 if let ModuleControlRequestFromModule::AgentCommands {
2151 started,
2152 finished,
2153 known,
2154 } = request
2155 {
2156 return self
2157 .agent_declarations
2158 .enqueue(
2159 self,
2160 ctx,
2161 frame,
2162 crate::resource_totals::Declaration {
2163 started,
2164 finished,
2165 known,
2166 },
2167 )
2168 .await;
2169 }
2170 let result =
2171 self.handle_module_control_request(ctx.connection_id, frame, request);
2172 log_slow_control_dispatch(
2173 dispatch_started_at,
2174 op,
2175 ctx.connection_id,
2176 corr,
2177 None,
2178 );
2179 return result;
2180 }
2181
2182 if is_known_module_request_op(&frame.body) {
2183 let is_agents = serde_json::from_slice::<ControlOpProbe>(&frame.body)
2184 .is_ok_and(|probe| probe.op == "supervisor.agent_commands");
2185 return Ok(vec![control_error_frame(
2186 &frame,
2187 if is_agents { "not_supervised" } else { "not_registered" },
2188 "module-originated control requests require the module's own registered connection",
2189 )?]);
2190 }
2191
2192 let request = match parse_client_control_request(&frame.body) {
2193 Ok(request) => request,
2194 Err((err, ControlRequestBodyError::UnknownOp)) => {
2195 return Ok(vec![control_error_frame(
2196 &frame,
2197 "unknown_control_op",
2198 format!("unknown client control op: {err}"),
2199 )?])
2200 }
2201 Err((err, ControlRequestBodyError::InvalidBody)) => {
2202 return Ok(vec![control_error_frame(
2203 &frame,
2204 "invalid_control_body",
2205 format!("malformed client control body: {err}"),
2206 )?])
2207 }
2208 };
2209 let op = client_control_request_op(&request);
2210 let corr = frame.header.corr;
2211 log_control_dispatch_arrival(op, ctx.connection_id, corr);
2212 let mut route_open_timings = (op == ops::ROUTE_OPEN).then(RouteOpenTimings::new);
2213 #[cfg(test)]
2214 if let Some(delay) = self.control_dispatch_delay {
2215 tokio::time::sleep(delay).await;
2216 }
2217 let result = self
2218 .handle_client_control_request(ctx, frame, request, &mut route_open_timings)
2219 .await;
2220 if let Some(timings) = route_open_timings.as_mut() {
2221 timings.finish();
2222 }
2223 log_slow_control_dispatch(
2224 dispatch_started_at,
2225 op,
2226 ctx.connection_id,
2227 corr,
2228 route_open_timings.as_ref(),
2229 );
2230 result
2231 }
2232 FrameType::Push => {
2233 let Some(endpoint) = self
2234 .forwarding
2235 .module_endpoint_for_connection(ctx.connection_id)
2236 .map_err(RouterError::Forwarding)?
2237 else {
2238 return Ok(vec![control_error_frame(
2239 &frame,
2240 "unsupported_control_frame",
2241 "client-originated channel-0 PUSH is not supported",
2242 )?]);
2243 };
2244 self.handle_status_update(endpoint, frame)
2245 }
2246 FrameType::Response | FrameType::Error
2247 if self
2248 .forwarding
2249 .module_endpoint_for_connection(ctx.connection_id)
2250 .map_err(RouterError::Forwarding)?
2251 .is_some() =>
2252 {
2253 self.handle_module_relay_response(ctx.connection_id, frame)
2254 }
2255 ty => Ok(vec![control_error_frame(
2256 &frame,
2257 "unsupported_control_frame",
2258 format!("unsupported channel-0 frame {ty:?}"),
2259 )?]),
2260 }
2261 }
2262
2263 pub fn cleanup_connection(
2264 &self,
2265 connection_id: ConnectionId,
2266 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2267 self.local_connections
2268 .lock()
2269 .unwrap_or_else(|p| p.into_inner())
2270 .remove(&connection_id);
2271 self.caller_endpoints
2272 .lock()
2273 .unwrap_or_else(|p| p.into_inner())
2274 .remove(&connection_id);
2275 self.build_class_warned
2276 .lock()
2277 .unwrap_or_else(|p| p.into_inner())
2278 .remove(&connection_id);
2279 self.observed_peers
2280 .lock()
2281 .unwrap_or_else(|p| p.into_inner())
2282 .remove(&connection_id);
2283 self.operator_apps
2284 .lock()
2285 .unwrap_or_else(|p| p.into_inner())
2286 .remove(&connection_id);
2287 self.forwarding
2288 .operator_confirms()
2289 .connection_closed(connection_id);
2290 self.cleanup_connection_with_end_reason(
2291 connection_id,
2292 RegistrationEndReason::ConnectionClosed,
2293 )
2294 }
2295
2296 fn cleanup_connection_with_end_reason(
2297 &self,
2298 connection_id: ConnectionId,
2299 requested_reason: RegistrationEndReason,
2300 ) -> Result<Vec<crate::registry::ModuleRegistration>, RegistryError> {
2301 self.agent_declarations.close(connection_id);
2302 let end_reason = if requested_reason == RegistrationEndReason::ConnectionClosed {
2303 self.registry
2304 .get_module_by_connection(connection_id)?
2305 .and_then(|registration| self.supervisor.get(®istration.manifest.module_id))
2306 .and_then(|module| module.registration_end_reason().ok().flatten())
2307 .unwrap_or(requested_reason)
2308 } else {
2309 requested_reason
2310 };
2311 let crash_closed = self
2312 .registry
2313 .get_module_by_connection(connection_id)?
2314 .and_then(|registration| {
2315 self.forwarding
2316 .module_endpoint_for_connection(connection_id)
2317 .ok()
2318 .flatten()
2319 .and_then(|endpoint| self.forwarding.endpoint_routes(endpoint).ok())
2320 .map(|routes| (registration.manifest.module_id, routes))
2321 });
2322 let crash_closed = crash_closed.map(|(module_id, routes)| {
2323 let terminal = match self.supervisor.get(&module_id) {
2324 None => false,
2325 Some(module) => match module.will_recover_after_connection_loss() {
2326 Ok(will_recover) => !will_recover,
2327 Err(err) => {
2328 warn!(
2329 %module_id,
2330 error = %err,
2331 "failed to read crash recovery verdict; reporting non-terminal conservatively"
2332 );
2333 false
2334 }
2335 },
2336 };
2337 let (reason, drained) = match self
2341 .forwarding
2342 .daemon_drain_result_for_connection(connection_id)
2343 {
2344 Ok(Some(drained)) => (RouteCloseReason::Restart, drained),
2345 Ok(None) => (RouteCloseReason::Crash, false),
2346 Err(err) => {
2347 warn!(error = %err, "failed to read daemon drain state; reporting crash conservatively");
2348 (RouteCloseReason::Crash, false)
2349 }
2350 };
2351 (module_id, routes, reason, drained, terminal)
2352 });
2353 let registrations = self.deregister_connection(connection_id, end_reason);
2354 let cleanup = if crash_closed.is_some() {
2355 self.forwarding.cleanup_connection_counted(connection_id)
2356 } else {
2357 self.forwarding
2363 .cleanup_connection(connection_id)
2364 .map(|released| crate::forwarding::ConnectionCleanup {
2365 released,
2366 abandoned_relays: 0,
2367 })
2368 };
2369 if let Some((module_id, routes, reason, drained, terminal)) = crash_closed {
2374 let abandoned = cleanup
2375 .as_ref()
2376 .map_or(0, |cleanup| cleanup.abandoned_relays);
2377 send_route_control_pushes(
2378 &self.forwarding,
2379 routes,
2380 ClientControlPush::RouteClosed {
2381 module_id,
2382 channels: Vec::new(),
2383 reason,
2384 drained,
2385 abandoned,
2386 excluded_subscriptions: 0,
2387 terminal: Some(terminal),
2388 },
2389 );
2390 }
2391 if let Ok(cleanup) = cleanup {
2392 self.emit_route_goodbyes(cleanup.released);
2393 }
2394 if matches!(®istrations, Ok(r) if !r.is_empty()) {
2398 crate::supervise::notify_registration_release();
2399 self.capability_evaluator.wake_deadline_loop();
2400 self.refresh_capability_requirements();
2401 }
2402 self.supervisor.remove_spawn_subscribers(connection_id);
2403 self.activity_streams
2404 .lock()
2405 .unwrap_or_else(|p| p.into_inner())
2406 .retain(|(id, _), task| {
2407 if *id == connection_id {
2408 task.abort();
2409 false
2410 } else {
2411 true
2412 }
2413 });
2414 self.hello_launch_nonces
2417 .lock()
2418 .unwrap_or_else(|poisoned| poisoned.into_inner())
2419 .forget(connection_id);
2420 self.scopes
2421 .write()
2422 .unwrap_or_else(|poisoned| poisoned.into_inner())
2423 .release_connection(connection_id);
2424 registrations
2425 }
2426
2427 pub(crate) fn handle_route_goodbye(
2428 &self,
2429 connection_id: ConnectionId,
2430 route_channel: u16,
2431 route_epoch: u32,
2432 ) -> Result<bool, RouterError> {
2433 debug!(
2434 connection_id = connection_id.get(),
2435 route_channel, route_epoch, "handling route GOODBYE"
2436 );
2437 let RouteRelease::Removed(released_route) = self
2438 .forwarding
2439 .release_client_route(connection_id, route_channel, route_epoch)
2440 .map_err(RouterError::Forwarding)?
2441 else {
2442 return Ok(false);
2443 };
2444 self.emit_route_goodbyes(vec![released_route]);
2445 Ok(true)
2446 }
2447
2448 fn emit_route_goodbyes(&self, released_routes: Vec<GoodbyeTarget>) {
2449 for released in released_routes {
2450 let frame = match Frame::build_with_version(
2451 released.negotiated_ver,
2452 FrameType::Goodbye,
2453 control_flags(),
2454 released.channel,
2455 released.epoch,
2456 0,
2457 Vec::new(),
2458 ) {
2459 Ok(frame) => frame,
2460 Err(err) => {
2461 warn!(
2462 route_channel = released.channel,
2463 error = %err,
2464 "failed to build route GOODBYE frame"
2465 );
2466 continue;
2467 }
2468 };
2469 if !released.close_on_delivery_failure() {
2470 crate::forwarding::send_module_route_goodbye(
2471 &self.counters,
2472 &released.sink,
2473 frame,
2474 released.module_id.as_deref(),
2475 "client route released",
2476 );
2477 continue;
2478 }
2479 if let Err(err) = released.sink.try_send(frame) {
2480 warn!(
2481 target_connection_id = released.connection_id.get(),
2482 route_channel = released.channel,
2483 error = %err,
2484 "route GOODBYE was not delivered to client; closing target connection"
2485 );
2486 if self
2487 .forwarding
2488 .escalate_client_delivery_failure(
2489 released.connection_id,
2490 released.channel,
2491 released.epoch,
2492 CloseReason::new(
2493 "route_goodbye_delivery_failed",
2494 format!(
2495 "failed to enqueue route GOODBYE for channel {}: {err}",
2496 released.channel
2497 ),
2498 ),
2499 crate::forwarding::UndeliveredFrame {
2500 module_id: released.module_id.as_deref(),
2501 sink: &released.sink,
2502 },
2503 )
2504 .unwrap_or(false)
2505 {
2506 self.counters.increment_goodbye_relay_client_failed();
2507 }
2508 }
2509 }
2510 }
2511
2512 fn send_abandoned_route_bind_goodbye(
2524 &self,
2525 module_sink: &crate::FrameSink,
2526 negotiated_ver: u8,
2527 module_channel: u16,
2528 module_epoch: u32,
2529 ) {
2530 let frame = match Frame::build_with_version(
2531 negotiated_ver,
2532 FrameType::Goodbye,
2533 control_flags(),
2534 module_channel,
2535 module_epoch,
2536 0,
2537 Vec::new(),
2538 ) {
2539 Ok(frame) => frame,
2540 Err(err) => {
2541 warn!(
2542 route_channel = module_channel,
2543 error = %err,
2544 "failed to build GOODBYE for abandoned route.bind"
2545 );
2546 return;
2547 }
2548 };
2549 crate::forwarding::send_module_route_goodbye(
2550 &self.counters,
2551 module_sink,
2552 frame,
2553 None,
2554 "abandoned route.bind",
2555 );
2556 }
2557
2558 fn handle_hello(
2559 &self,
2560 connection_id: ConnectionId,
2561 sink: Option<crate::FrameSink>,
2562 frame: Frame,
2563 ) -> Result<Vec<Frame>, RouterError> {
2564 debug!(
2565 connection_id = connection_id.get(),
2566 corr = frame.header.corr,
2567 "handling HELLO"
2568 );
2569 if self
2573 .registry
2574 .get_module_by_connection(connection_id)
2575 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2576 .is_some()
2577 {
2578 return Ok(vec![control_error_frame(
2579 &frame,
2580 "invalid_hello",
2581 "connection is already registered as a module",
2582 )?]);
2583 }
2584 let hello_value = match serde_json::from_slice::<serde_json::Value>(&frame.body) {
2585 Ok(value) => value,
2586 Err(err) => {
2587 return Ok(vec![control_error_frame(
2588 &frame,
2589 "invalid_hello",
2590 format!("malformed HELLO body: {err}"),
2591 )?])
2592 }
2593 };
2594 if let Err(err) = validate_hello_display_name(&hello_value) {
2595 return Ok(vec![control_error_frame(
2596 &frame,
2597 "invalid_manifest",
2598 err.to_string(),
2599 )?]);
2600 }
2601 if let Err(err) = validate_hello_capability_grammar(&hello_value) {
2602 return Ok(vec![control_error_frame(
2603 &frame,
2604 "invalid_capability_grammar",
2605 err.to_string(),
2606 )?]);
2607 }
2608 if let Err(err) = validate_hello_self_signal_declarations(&hello_value) {
2609 return Ok(vec![control_error_frame(
2610 &frame,
2611 "invalid_manifest",
2612 err.to_string(),
2613 )?]);
2614 }
2615 if let Err(err) = validate_hello_event_declarations(&hello_value) {
2616 return Ok(vec![control_error_frame(
2617 &frame,
2618 "invalid_event_declaration",
2619 err.to_string(),
2620 )?]);
2621 }
2622 if let Err(err) = validate_hello_rich_ui_descriptor(&hello_value) {
2623 return Ok(vec![control_error_frame(
2624 &frame,
2625 "invalid_manifest",
2626 err.to_string(),
2627 )?]);
2628 }
2629 if let Some(provenance) = hello_value
2630 .get("manifest")
2631 .and_then(|manifest| manifest.get("provenance"))
2632 {
2633 if let Err(err) = serde_json::from_value::<ManifestProvenance>(provenance.clone()) {
2634 return Ok(vec![control_error_frame(
2635 &frame,
2636 "invalid_manifest",
2637 format!("malformed manifest provenance: {err}"),
2638 )?]);
2639 }
2640 }
2641 if let Err(reason) = validate_hello_role_declarations(&hello_value) {
2642 let module_id = hello_value["manifest"]["module_id"]
2643 .as_str()
2644 .unwrap_or("<unknown>");
2645 warn!(module_id, %reason, "dropping whole role_declarations map from HELLO");
2646 }
2647 if let Err(reason) = validate_hello_flow_operations(&hello_value) {
2648 let module_id = hello_value["manifest"]["module_id"]
2649 .as_str()
2650 .unwrap_or("<unknown>");
2651 warn!(module_id, %reason, "dropping flow_operations from HELLO");
2652 }
2653 if let Err(reason) = validate_hello_confirm_operations(&hello_value) {
2654 let module_id = hello_value["manifest"]["module_id"]
2655 .as_str()
2656 .unwrap_or("<unknown>");
2657 warn!(module_id, %reason, "dropping confirm_operations from HELLO");
2658 }
2659 let hello = match serde_json::from_value::<ModuleHelloBody>(hello_value) {
2660 Ok(hello) => hello,
2661 Err(err) => {
2662 return Ok(vec![control_error_frame(
2663 &frame,
2664 "invalid_hello",
2665 format!("malformed HELLO body: {err}"),
2666 )?])
2667 }
2668 };
2669
2670 if hello.protocol_ver != hello.manifest.protocol_ver {
2671 return Ok(vec![control_error_frame(
2672 &frame,
2673 "invalid_manifest",
2674 format!(
2675 "HELLO protocol_ver {} does not match manifest protocol_ver {}",
2676 hello.protocol_ver, hello.manifest.protocol_ver
2677 ),
2678 )?]);
2679 }
2680
2681 if hello.manifest.module_id.trim().is_empty() {
2682 return Ok(vec![control_error_frame(
2683 &frame,
2684 "invalid_manifest",
2685 "manifest module_id must not be empty",
2686 )?]);
2687 }
2688
2689 let negotiated_ver = match negotiate_version(hello.protocol_ver) {
2690 Ok(negotiated_ver) => negotiated_ver,
2691 Err(message) => {
2692 return Ok(vec![control_error_frame(
2693 &frame,
2694 "version_unsupported",
2695 message,
2696 )?])
2697 }
2698 };
2699
2700 let swap_admission = self
2708 .supervisor
2709 .swap_hello_admission(&hello.manifest.module_id, hello.launch_nonce.as_deref());
2710 if swap_admission == SwapHelloAdmission::Refused {
2711 warn!(
2712 module_id = %hello.manifest.module_id,
2713 connection_id = connection_id.get(),
2714 "HELLO refused: a swap is open for this module_id and the launch nonce is not one the supervisor minted for it"
2715 );
2716 return Ok(vec![control_error_frame(
2717 &frame,
2718 "swap_token_invalid",
2719 format!(
2720 "module_id '{}' is being swapped; HELLO without the swap candidate's launch nonce is rejected",
2721 hello.manifest.module_id
2722 ),
2723 )?]);
2724 }
2725 let swap_candidate = swap_admission == SwapHelloAdmission::Candidate;
2726
2727 let verification = if swap_candidate {
2732 ReservedHelloVerification {
2733 nonce_verified: true,
2734 prefix_owner: None,
2735 }
2736 } else {
2737 match self.supervisor.reserved_hello_verification(
2738 &hello.manifest.module_id,
2739 hello.launch_nonce.as_deref(),
2740 ) {
2741 Ok(verified) => verified,
2742 Err(rejection) => {
2743 let message = match rejection {
2744 ReservedHelloRejection::Exact { module_id } => format!(
2745 "module_id '{module_id}' is reserved; HELLO without a valid launch nonce is rejected"
2746 ),
2747 ReservedHelloRejection::Prefix {
2748 prefix,
2749 owner_module_id,
2750 } => format!(
2751 "module_id '{}' matches reserved prefix '{prefix}' owned by '{owner_module_id}'; HELLO without the owner launch nonce is rejected",
2752 hello.manifest.module_id
2753 ),
2754 };
2755 return Ok(vec![control_error_frame(
2756 &frame,
2757 "reserved_module",
2758 message,
2759 )?]);
2760 }
2761 }
2762 };
2763
2764 let reserved_capability_refusals = self.capability_evaluator.reserved_hello_refusals(
2765 &hello.manifest.module_id,
2766 hello.manifest.capabilities.as_ref(),
2767 );
2768 if let Some(refusal) = reserved_capability_refusals.first() {
2769 let capability = refusal.capability.clone();
2770 let bound_module = refusal.claimants[0].clone();
2771 log_duplicate_claim_events(reserved_capability_refusals);
2772 return Ok(vec![control_error_frame(
2773 &frame,
2774 "reserved_capability",
2775 format!(
2776 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
2777 capability, bound_module, hello.manifest.module_id
2778 ),
2779 )?]);
2780 }
2781
2782 if self
2785 .forwarding
2786 .connection_has_client_routes(connection_id)
2787 .map_err(RouterError::Forwarding)?
2788 {
2789 return Ok(vec![control_error_frame(
2790 &frame,
2791 "invalid_hello",
2792 "connection has open client routes and cannot also register as a module",
2793 )?]);
2794 }
2795
2796 self.hello_launch_nonces
2801 .lock()
2802 .unwrap_or_else(|poisoned| poisoned.into_inner())
2803 .record(connection_id, hello.launch_nonce.as_deref());
2804 let control_ops = effective_module_control_ops(hello.control_ops);
2805 let hello_ack = self.build_hello_ack(&frame, negotiated_ver, &hello.manifest.module_id)?;
2808 if swap_candidate {
2809 return self.register_swap_candidate(
2810 connection_id,
2811 sink,
2812 &frame,
2813 hello.manifest,
2814 negotiated_ver,
2815 control_ops,
2816 hello_ack,
2817 );
2818 }
2819 let registration = match self
2826 .registry
2827 .register_unpublished_verified_with_control_ops(
2828 hello.manifest,
2829 negotiated_ver,
2830 connection_id,
2831 control_ops,
2832 verification.nonce_verified,
2833 ) {
2834 Ok(registration) => registration,
2835 Err(RegistryError::DuplicateModuleId { module_id }) => {
2836 return Ok(vec![control_error_frame(
2837 &frame,
2838 "duplicate_module_id",
2839 format!(
2840 "module_id '{module_id}' is already registered; duplicate HELLO rejected"
2841 ),
2842 )?])
2843 }
2844 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
2845 return Ok(vec![control_error_frame(
2846 &frame,
2847 "invalid_module_id",
2848 err.to_string(),
2849 )?])
2850 }
2851 Err(err) => {
2852 return Ok(vec![control_error_frame(
2853 &frame,
2854 "registry_error",
2855 err.to_string(),
2856 )?])
2857 }
2858 };
2859 #[cfg(test)]
2860 self.hold_between_registry_and_forwarding();
2861
2862 let reply = if let Some(sink) = sink {
2863 let concurrency = manifest_concurrency(®istration.manifest);
2879 if let Err(err) = self.forwarding.register_module_connection_acked(
2880 connection_id,
2881 registration.manifest.module_id.clone(),
2882 negotiated_ver,
2883 concurrency,
2884 sink,
2885 crate::forwarding::ModuleHelloAck {
2886 frame: hello_ack,
2887 verified_prefix_owner: verification.prefix_owner,
2888 },
2889 ) {
2890 if matches!(
2895 self.deregister_connection(
2896 connection_id,
2897 RegistrationEndReason::RegistrationFailed,
2898 ),
2899 Ok(r) if !r.is_empty()
2900 ) {
2901 crate::supervise::notify_registration_release();
2902 }
2903 return Ok(vec![control_error_frame(
2904 &frame,
2905 if matches!(err, ForwardingError::ConnectionRoleConflict { .. }) {
2906 "invalid_hello"
2907 } else {
2908 forwarding_error_code(&err)
2909 },
2910 err.to_string(),
2911 )?]);
2912 }
2913 Vec::new()
2914 } else {
2915 vec![hello_ack]
2918 };
2919
2920 if !self
2925 .registry
2926 .publish_registration(connection_id)
2927 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
2928 {
2929 debug!(
2930 module_id = %registration.manifest.module_id,
2931 connection_id = connection_id.get(),
2932 "registration was removed before it could be published"
2933 );
2934 }
2935
2936 if manifest_concurrency_was_defaulted(&frame.body, ®istration.manifest) {
2945 info!(
2946 module_id = %registration.manifest.module_id,
2947 "management surface registered with DEFAULTED concurrency=module_managed (manifest predates the field; declare the real lane)"
2948 );
2949 }
2950
2951 self.apply_registration_capabilities(®istration);
2952
2953 self.supervisor
2954 .record_activity_registration(®istration.manifest.module_id);
2955
2956 info!(
2957 module_id = %registration.manifest.module_id,
2958 module_version = %registration.manifest.module_version,
2959 negotiated_ver,
2960 routable_provider = manifest_provides_routable_role(®istration.manifest),
2961 connection_id = connection_id.get(),
2962 "module registered"
2963 );
2964
2965 Ok(reply)
2966 }
2967
2968 #[allow(clippy::too_many_arguments)]
2985 fn register_swap_candidate(
2986 &self,
2987 connection_id: ConnectionId,
2988 sink: Option<crate::FrameSink>,
2989 frame: &Frame,
2990 manifest: ModuleManifest,
2991 negotiated_ver: u8,
2992 control_ops: Vec<String>,
2993 hello_ack: Frame,
2994 ) -> Result<Vec<Frame>, RouterError> {
2995 let module_id = manifest.module_id.clone();
2996 let registration = match self.registry.register_unpublished_verified_candidate_with_control_ops(
2997 manifest,
2998 negotiated_ver,
2999 connection_id,
3000 control_ops,
3001 ) {
3002 Ok(registration) => registration,
3003 Err(RegistryError::DuplicateModuleId { module_id }) => {
3004 return Ok(vec![control_error_frame(
3005 frame,
3006 "duplicate_module_id",
3007 format!(
3008 "module_id '{module_id}' already has a swap candidate registered; duplicate HELLO rejected"
3009 ),
3010 )?])
3011 }
3012 Err(err @ RegistryError::PathHazardModuleId { .. }) => {
3013 return Ok(vec![control_error_frame(
3014 frame,
3015 "invalid_module_id",
3016 err.to_string(),
3017 )?])
3018 }
3019 Err(err) => {
3020 return Ok(vec![control_error_frame(
3021 frame,
3022 "registry_error",
3023 err.to_string(),
3024 )?])
3025 }
3026 };
3027 #[cfg(test)]
3028 self.hold_between_registry_and_forwarding();
3029 let reply = if let Some(sink) = sink {
3030 let concurrency = manifest_concurrency(®istration.manifest);
3034 if let Err(err) = self.forwarding.register_candidate_module_connection_acked(
3035 connection_id,
3036 module_id.clone(),
3037 negotiated_ver,
3038 concurrency,
3039 sink,
3040 hello_ack,
3041 ) {
3042 if matches!(
3043 self.deregister_connection(
3044 connection_id,
3045 RegistrationEndReason::RegistrationFailed,
3046 ),
3047 Ok(r) if !r.is_empty()
3048 ) {
3049 crate::supervise::notify_registration_release();
3050 }
3051 return Ok(vec![control_error_frame(
3052 frame,
3053 forwarding_error_code(&err),
3054 err.to_string(),
3055 )?]);
3056 }
3057 Vec::new()
3058 } else {
3059 vec![hello_ack]
3060 };
3061 if !self
3062 .registry
3063 .publish_registration(connection_id)
3064 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3065 {
3066 debug!(
3067 module_id = %module_id,
3068 connection_id = connection_id.get(),
3069 "swap candidate registration was removed before it could be published"
3070 );
3071 }
3072 self.supervisor.mark_swap_candidate_admitted(&module_id);
3073 info!(
3074 module_id = %module_id,
3075 module_version = %registration.manifest.module_version,
3076 negotiated_ver,
3077 ready = registration.ready,
3078 connection_id = connection_id.get(),
3079 "swap candidate registered; not routable until cutover"
3080 );
3081 Ok(reply)
3082 }
3083
3084 fn build_hello_ack(
3085 &self,
3086 frame: &Frame,
3087 negotiated_ver: u8,
3088 module_id: &str,
3089 ) -> Result<Frame, RouterError> {
3090 let ack = ModuleHelloAckBody {
3091 negotiated_ver,
3092 subc_ops: module_subc_ops(),
3093 subc_capabilities: self.subc_capabilities.as_ref().to_vec(),
3094 storage: self
3095 .storage_config
3096 .as_ref()
3097 .map(|cfg| cfg.descriptor_for(module_id)),
3098 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
3099 };
3100 let body = serde_json::to_vec(&ack).map_err(|err| {
3101 RouterError::backend(
3102 0,
3103 frame.header.corr,
3104 format!("failed to encode HELLO_ACK: {err}"),
3105 )
3106 })?;
3107
3108 Frame::build_with_version(
3109 negotiated_ver,
3110 FrameType::HelloAck,
3111 control_flags(),
3112 0,
3113 0,
3114 frame.header.corr,
3115 body,
3116 )
3117 .map_err(RouterError::FrameBuild)
3118 }
3119
3120 async fn handle_client_control_request(
3121 &self,
3122 ctx: &RouteCtx,
3123 frame: Frame,
3124 request: ClientControlRequest,
3125 route_open_timings: &mut Option<RouteOpenTimings>,
3126 ) -> Result<Vec<Frame>, RouterError> {
3127 if let Some(operation) = privileged_operation(&request) {
3128 let caller = self.resolve_caller(ctx.connection_id).await;
3129 let op = client_control_request_op(&request);
3130 let text = prompt_text(operation, &caller);
3131 match self.privileged_operator_prompt {
3132 PrivilegedOperatorPrompt::Log => {
3133 info!(target: "subc_daemon::privileged_prompt", operation = op,
3134 target = ?operation, caller = %caller.description(), summary = %text,
3135 "privileged request would prompt the operator");
3136 }
3137 PrivilegedOperatorPrompt::Enforce => {
3138 let key =
3139 crate::operator_confirm::DeclineKey::for_caller(&caller, ctx.connection_id);
3140 let stamp = self
3141 .operator_apps
3142 .lock()
3143 .unwrap_or_else(|p| p.into_inner())
3144 .get(&ctx.connection_id)
3145 .cloned();
3146 let shortcut = crate::operator_confirm::ShortcutContext::eligible_stamp(
3147 &self.operator_app_policy,
3148 &self.launch_table,
3149 stamp.as_ref(),
3150 )
3151 .then(|| self.shortcut_context(stamp, None, None));
3152 let outcome = self
3153 .forwarding
3154 .operator_confirms()
3155 .approve_privileged_with_shortcut(
3156 ctx,
3157 &frame,
3158 op,
3159 text,
3160 caller.description(),
3161 key,
3162 shortcut,
3163 )
3164 .await;
3165 if outcome.code != "confirmed" {
3166 return Ok(vec![outcome.frame(&frame)]);
3167 }
3168 }
3169 }
3170 }
3171 match request {
3172 ClientControlRequest::DaemonShutdownIntent { intent } => {
3173 if intent.as_str() != Some("restart") {
3174 return Ok(vec![control_error_frame(
3175 &frame,
3176 "invalid_params",
3177 "intent must be 'restart'",
3178 )?]);
3179 }
3180 let response = ClientControlResponse::DaemonShutdownIntent {
3181 intent: "restart".to_owned(),
3182 expires_at_ms: self.supervisor.announce_daemon_restart(),
3183 };
3184 Ok(vec![control_response_body_frame(
3185 &frame,
3186 &response,
3187 "ClientControlResponse::DaemonShutdownIntent",
3188 )?])
3189 }
3190 ClientControlRequest::ServerDescribe {} => self.handle_server_describe(frame),
3191 ClientControlRequest::CatalogList { module_id } => {
3192 self.handle_catalog_list(frame, module_id)
3193 }
3194 ClientControlRequest::RouteOpen {
3195 target,
3196 identity,
3197 consumer_identity,
3198 consumer_capabilities,
3199 role_versions,
3200 busy_refusals,
3201 admission_facts,
3202 scope,
3203 ..
3204 } => {
3205 self.handle_route_open(
3206 ctx,
3207 frame,
3208 RouteOpenRequest {
3209 target,
3210 identity,
3211 consumer_identity,
3212 consumer_capabilities,
3213 role_versions,
3214 busy_refusals,
3215 admission_facts,
3216 scope,
3217 },
3218 route_open_timings.get_or_insert_with(RouteOpenTimings::new),
3219 )
3220 .await
3221 }
3222 ClientControlRequest::RoutePoll {
3223 route_channel,
3224 route_epoch,
3225 kind,
3226 } => self.handle_route_poll(ctx, frame, route_channel, route_epoch, kind),
3227 ClientControlRequest::SupervisorList {} => {
3228 let Ok(permit) = Arc::clone(&self.list_admission).try_acquire_owned() else {
3229 self.agent_declarations.refuse(
3230 self,
3231 ctx,
3232 control_error_frame(
3233 &frame,
3234 "busy",
3235 "supervisor list work is at capacity; retry",
3236 )?,
3237 );
3238 return Ok(Vec::new());
3239 };
3240 let handler = self.clone();
3242 let sink = ctx.egress.clone();
3243 tokio::spawn(
3244 async move {
3245 let _permit = permit;
3246 let result = handler.handle_supervisor_list(frame.clone()).await;
3247 let replies = match result {
3248 Ok(replies) => replies,
3249 Err(error) => {
3250 control_error_frame(&frame, "backend_error", error.to_string())
3251 .into_iter()
3252 .collect()
3253 }
3254 };
3255 for reply in replies {
3256 if !matches!(
3257 tokio::time::timeout(Duration::from_millis(200), sink.send(reply))
3258 .await,
3259 Ok(Ok(()))
3260 ) {
3261 break;
3262 }
3263 }
3264 }
3265 .with_current_subscriber(),
3266 );
3267 Ok(Vec::new())
3268 }
3269 ClientControlRequest::SupervisorSpawnSnapshot {} => {
3270 self.handle_supervisor_spawn_snapshot(frame)
3271 }
3272 ClientControlRequest::SupervisorSpawnSubscribe { since } => {
3273 self.handle_supervisor_spawn_subscribe(ctx, frame, since)
3274 }
3275 ClientControlRequest::SupervisorRestart {
3276 module_id,
3277 drain_timeout_ms,
3278 } => {
3279 self.log_supervisor_request_received(
3280 ctx,
3281 frame.header.corr,
3282 ops::SUPERVISOR_RESTART,
3283 Some(&module_id),
3284 None,
3285 )?;
3286 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3287 self.handle_supervisor_restart(frame, module_id, drain_timeout_ms, requester)
3288 .await
3289 }
3290 ClientControlRequest::SupervisorSwap {
3291 module_id,
3292 ready_timeout_ms,
3293 } => {
3294 self.log_supervisor_request_received(
3295 ctx,
3296 frame.header.corr,
3297 ops::SUPERVISOR_SWAP,
3298 Some(&module_id),
3299 None,
3300 )?;
3301 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3302 self.handle_supervisor_swap(frame, module_id, ready_timeout_ms, requester)
3303 .await
3304 }
3305 ClientControlRequest::SupervisorReload { module_id } => {
3306 self.log_supervisor_request_received(
3307 ctx,
3308 frame.header.corr,
3309 ops::SUPERVISOR_RELOAD,
3310 Some(&module_id),
3311 None,
3312 )?;
3313 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3314 self.handle_supervisor_reload(frame, module_id, requester)
3315 .await
3316 }
3317 ClientControlRequest::SupervisorRescan { preview } => {
3318 if !preview {
3319 self.log_supervisor_request_received(
3320 ctx,
3321 frame.header.corr,
3322 ops::SUPERVISOR_RESCAN,
3323 None,
3324 None,
3325 )?;
3326 }
3327 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3328 let operation = self.handle_supervisor_rescan(
3329 frame,
3330 if preview {
3331 RescanMode::Preview
3332 } else {
3333 RescanMode::Execute
3334 },
3335 requester,
3336 );
3337 self.observe_control_wait(ctx, None, ops::SUPERVISOR_RESCAN, operation)
3338 .await
3339 }
3340 ClientControlRequest::SupervisorRescanChecked {
3341 preview,
3342 candidate,
3343 expect,
3344 } => {
3345 self.observe_control_wait(
3346 ctx,
3347 None,
3348 ops::SUPERVISOR_RESCAN,
3349 self.handle_supervisor_rescan_checked(ctx, frame, preview, candidate, expect),
3350 )
3351 .await
3352 }
3353 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
3354 self.log_supervisor_request_received(
3355 ctx,
3356 frame.header.corr,
3357 ops::SUPERVISOR_RELEASE_RESERVED,
3358 Some(&module_id),
3359 None,
3360 )?;
3361 self.handle_supervisor_release_reserved(frame, module_id)
3362 .await
3363 }
3364 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
3365 self.log_supervisor_request_received(
3366 ctx,
3367 frame.header.corr,
3368 ops::SUPERVISOR_SET_ENABLED,
3369 Some(&module_id),
3370 Some(enabled),
3371 )?;
3372 let requester = self.supervisor_requester(ctx, frame.header.corr)?;
3373 self.handle_supervisor_set_enabled(frame, module_id, enabled, requester)
3374 .await
3375 }
3376 ClientControlRequest::SupervisorHealthProbe { module_id } => {
3377 self.handle_supervisor_health_probe(ctx, frame, module_id)
3378 .await
3379 }
3380 ClientControlRequest::SupervisorHealth {} => self.handle_supervisor_health(frame),
3381 ClientControlRequest::SupervisorRoutes { module_id } => {
3382 self.handle_supervisor_routes(frame, module_id)
3383 }
3384 ClientControlRequest::SupervisorProvenance { module_id } => {
3385 self.handle_supervisor_provenance(ctx, frame, module_id)
3386 .await
3387 }
3388 ClientControlRequest::SupervisorStderrTail {
3389 module_id,
3390 max_lines,
3391 max_bytes,
3392 } => self.handle_supervisor_stderr_tail(frame, module_id, max_lines, max_bytes),
3393 ClientControlRequest::SupervisorTerminals { module_id } => {
3394 self.handle_supervisor_terminals(ctx, frame, module_id)
3395 .await
3396 }
3397 }
3398 }
3399
3400 fn log_supervisor_request_received(
3401 &self,
3402 ctx: &RouteCtx,
3403 corr: u64,
3404 op: &'static str,
3405 module_id: Option<&str>,
3406 enabled: Option<bool>,
3407 ) -> Result<(), RouterError> {
3408 let caller = self.supervisor_requester(ctx, corr)?;
3409 let caller = principal_label(&caller);
3410 info!(
3413 target: "control",
3414 op,
3415 module_id,
3416 enabled,
3417 connection_id = ctx.connection_id.get(),
3418 caller = %caller,
3419 "supervisor request received"
3420 );
3421 Ok(())
3422 }
3423
3424 fn supervisor_requester(&self, ctx: &RouteCtx, corr: u64) -> Result<Principal, RouterError> {
3425 Ok(self
3426 .registry
3427 .get_module_by_connection(ctx.connection_id)
3428 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
3429 .map(|registration| Principal::Reserved {
3430 module_id: registration.manifest.module_id,
3431 })
3432 .unwrap_or(Principal::Direct))
3433 }
3434
3435 fn handle_module_control_request(
3436 &self,
3437 connection_id: ConnectionId,
3438 frame: Frame,
3439 request: ModuleControlRequestFromModule,
3440 ) -> Result<Vec<Frame>, RouterError> {
3441 match request {
3442 ModuleControlRequestFromModule::CatalogUpdate {
3443 provides,
3444 capabilities,
3445 ready,
3446 } => self.handle_catalog_update(connection_id, frame, provides, capabilities, ready),
3447 ModuleControlRequestFromModule::LiveRoots {} => {
3448 let registered = self
3449 .registry
3450 .get_module_by_connection(connection_id)
3451 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3452 let Some(registration) = registered else {
3453 return Ok(vec![control_error_frame(&frame, "not_registered", "supervisor.live_roots requires an active module registration owned by this connection")?]);
3454 };
3455 let response = self
3456 .forwarding
3457 .live_roots(®istration.manifest.module_id)
3458 .map_err(RouterError::Forwarding)?;
3459 Ok(vec![control_response_body_frame(
3460 &frame,
3461 &response,
3462 "ModuleControlResponseToModule::LiveRoots",
3463 )?])
3464 }
3465 ModuleControlRequestFromModule::AgentCommands {
3466 started,
3467 finished,
3468 known,
3469 } => self.handle_agent_commands(connection_id, frame, started, finished, known),
3470 ModuleControlRequestFromModule::ScopeSync { generation, scopes } => {
3471 self.handle_scope_sync(connection_id, frame, generation, scopes)
3472 }
3473 ModuleControlRequestFromModule::ScopeApply {
3474 generation,
3475 upsert,
3476 end,
3477 } => self.handle_scope_change(connection_id, frame, generation, upsert, Some(end)),
3478 ModuleControlRequestFromModule::ScopeDescribe { owner, scope_ref } => {
3479 self.handle_scope_describe(connection_id, frame, owner, scope_ref)
3480 }
3481 _ => Ok(vec![control_error_frame(
3482 &frame,
3483 "unsupported_control_frame",
3484 "unsupported module control operation",
3485 )?]),
3486 }
3487 }
3488
3489 fn handle_scope_served_by(
3490 &self,
3491 connection_id: ConnectionId,
3492 frame: Frame,
3493 ) -> Result<Vec<Frame>, RouterError> {
3494 let registered = self
3495 .registry
3496 .get_module_by_connection(connection_id)
3497 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3498 let Some(registered) = registered else {
3499 return Ok(vec![control_error_frame(
3500 &frame,
3501 "not_registered",
3502 "scope.served_by requires an active module registration owned by this connection",
3503 )?]);
3504 };
3505 let request: subc_control::ScopeServedByRequest = match serde_json::from_slice(&frame.body)
3506 {
3507 Ok(request) => request,
3508 Err(err) => {
3509 return Ok(vec![control_error_frame(
3510 &frame,
3511 "invalid_control_body",
3512 format!("malformed scope.served_by body: {err}"),
3513 )?])
3514 }
3515 };
3516 let answer = self
3517 .scopes
3518 .read()
3519 .unwrap_or_else(|poisoned| poisoned.into_inner())
3520 .served_by(
3521 ®istered.manifest.module_id,
3522 &request.scope_ref,
3523 request.scope_epoch,
3524 &request.module_id,
3525 );
3526 let reply = subc_control::ScopeServedByReply::new(
3527 answer,
3528 self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3529 );
3530 Ok(vec![control_response_body_frame(
3531 &frame,
3532 &reply,
3533 "ScopeServedByReply",
3534 )?])
3535 }
3536
3537 fn handle_operator_confirm(
3538 &self,
3539 ctx: &RouteCtx,
3540 frame: Frame,
3541 ) -> Result<Vec<Frame>, RouterError> {
3542 use crate::operator_confirm::{audit, Outcome};
3543 let registration = self
3544 .registry
3545 .get_module_by_connection(ctx.connection_id)
3546 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3547 let Some(registration) = registration else {
3548 let outcome = Outcome::refusal("not_registered");
3549 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3550 return Ok(vec![outcome.frame(&frame)]);
3551 };
3552 let request = match serde_json::from_slice::<OperatorConfirmRequest>(&frame.body) {
3553 Ok(request) => request,
3554 Err(_) => {
3555 let outcome = Outcome::refusal("invalid_control_body");
3556 audit("", "", "", &outcome, Duration::ZERO, Duration::ZERO, false);
3557 return Ok(vec![outcome.frame(&frame)]);
3558 }
3559 };
3560 let declared = matches!(
3561 request.kind,
3562 Some(subc_protocol::session::OperatorConfirmKind::Operation)
3563 ) && request.operation.as_ref().is_some_and(|operation| {
3564 registration
3565 .manifest
3566 .confirm_operations
3567 .as_ref()
3568 .is_some_and(|operations| operations.contains(operation))
3569 });
3570 let module_id = registration.manifest.module_id;
3571 let nonce = self
3575 .hello_launch_nonces
3576 .lock()
3577 .unwrap_or_else(|p| p.into_inner())
3578 .nonce(ctx.connection_id)
3579 .map(str::to_owned);
3580 let nonce_proven = nonce.as_deref().is_some_and(|nonce| {
3581 self.supervisor
3582 .spawned_consumer_authorized(&module_id, nonce)
3583 });
3584 let confirms = self.forwarding.operator_confirms();
3585 self.forwarding
3586 .with_operator_route(
3587 ctx.connection_id,
3588 request.route_channel,
3589 request.route_epoch,
3590 |binding| {
3591 let shortcut = binding
3592 .filter(|binding| {
3593 nonce_proven
3594 && crate::operator_confirm::valid_summary(&request.summary)
3595 && declared
3596 && crate::operator_confirm::ShortcutContext::route_eligible(
3597 &self.operator_app_policy,
3598 &self.launch_table,
3599 binding,
3600 )
3601 })
3602 .map(|binding| {
3603 let phone = crate::operator_confirm::ShortcutContext::phone_fingerprint(
3604 &self.operator_app_policy,
3605 binding,
3606 );
3607 let stamp = if phone.is_some() {
3608 None
3609 } else {
3610 binding.operator_app.clone()
3611 };
3612 self.shortcut_context(
3613 stamp,
3614 phone,
3615 Some((request.route_channel, request.route_epoch)),
3616 )
3617 });
3618 confirms.admit_with_shortcut(
3619 ctx,
3620 frame,
3621 module_id,
3622 nonce_proven,
3623 request,
3624 binding,
3625 shortcut,
3626 )
3627 },
3628 )
3629 .map_err(RouterError::Forwarding)
3630 }
3631
3632 fn shortcut_context(
3633 &self,
3634 stamp: Option<crate::operator_apps::OperatorAppStamp>,
3635 phone_fingerprint: Option<String>,
3636 route: Option<(u16, u32)>,
3637 ) -> crate::operator_confirm::ShortcutContext {
3638 crate::operator_confirm::ShortcutContext {
3639 ledger: self.activity.clone(),
3640 forwarding: self.forwarding.clone(),
3641 launches: self.launch_table.clone(),
3642 stamp,
3643 phone_fingerprint,
3644 route,
3645 connections: self.operator_apps.clone(),
3646 #[cfg(test)]
3647 hooks: Default::default(),
3648 }
3649 }
3650
3651 #[cfg(any(test, feature = "test-support"))]
3653 pub fn with_test_phone_shortcut(mut self, enabled: bool) -> Self {
3654 self.operator_app_policy.phone_enabled = enabled;
3655 self
3656 }
3657
3658 pub(crate) fn prepare_scope_records(scopes: Vec<ScopeRecord>) -> ScopeRecords {
3661 let mut scopes = ScopeRecords::from(scopes);
3662 for (index, record) in scopes.records.iter_mut().enumerate() {
3663 if let Some(refusal) = Self::canonicalize_workdir(record) {
3664 scopes.refusals.insert(index, refusal);
3665 }
3666 }
3667 scopes
3668 }
3669
3670 fn canonicalize_workdir(record: &mut ScopeRecord) -> Option<crate::scopes::RecordRefusal> {
3671 let workdir = record.workdir.as_mut()?;
3672 let path = std::path::Path::new(workdir);
3673 if !path.is_absolute() {
3674 return Some((
3675 error_codes::INVALID_WORKDIR,
3676 "workdir must be absolute".to_string(),
3677 ));
3678 }
3679 match ProjectRootId::from_path_allowing_missing(path) {
3680 Ok(root) => match root.as_path().to_str() {
3681 Some(canonical) => {
3682 *workdir = canonical.to_string();
3683 None
3684 }
3685 None => Some((
3686 error_codes::INVALID_WORKDIR,
3687 "canonical workdir is not UTF-8".to_string(),
3688 )),
3689 },
3690 Err(error) => Some((error_codes::INVALID_WORKDIR, error.to_string())),
3691 }
3692 }
3693
3694 fn handle_scope_sync(
3698 &self,
3699 connection_id: ConnectionId,
3700 frame: Frame,
3701 generation: u64,
3702 scopes: Vec<ScopeRecord>,
3703 ) -> Result<Vec<Frame>, RouterError> {
3704 self.handle_scope_change(connection_id, frame, generation, scopes, None)
3705 }
3706
3707 fn handle_scope_change(
3708 &self,
3709 connection_id: ConnectionId,
3710 frame: Frame,
3711 generation: u64,
3712 scopes: Vec<ScopeRecord>,
3713 end: Option<Vec<ScopeEnd>>,
3714 ) -> Result<Vec<Frame>, RouterError> {
3715 let is_apply = end.is_some();
3716 let Some(registration) = self
3717 .registry
3718 .get_module_by_connection(connection_id)
3719 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3720 else {
3721 return Ok(vec![control_error_frame(
3722 &frame,
3723 "not_registered",
3724 "scope.sync and scope.apply require an active module registration owned by this connection",
3725 )?]);
3726 };
3727 let owner = registration.manifest.module_id;
3728 let scopes = Self::prepare_scope_records(scopes);
3729 let current_nonce = self.supervisor.spawn_launch_nonce_for(&owner);
3730 let is_current_launch = |connection: ConnectionId| {
3731 self.hello_launch_nonces
3732 .lock()
3733 .unwrap_or_else(|poisoned| poisoned.into_inner())
3734 .presented(connection, current_nonce.as_deref())
3735 };
3736 let mut table = self
3741 .scopes
3742 .write()
3743 .unwrap_or_else(|poisoned| poisoned.into_inner());
3744 let outcome = match end {
3745 Some(end) => table.apply(
3746 &owner,
3747 connection_id,
3748 is_current_launch,
3749 generation,
3750 scopes,
3751 end,
3752 ),
3753 None => table.sync(&owner, connection_id, is_current_launch, generation, scopes),
3754 };
3755 let drained = match &outcome {
3756 Ok(applied) => self.publish_scope_changes(&applied.tag_changes, &applied.expired)?,
3757 Err(_) => Vec::new(),
3758 };
3759 let has_deadlines = outcome.is_ok() && table.has_deadlines();
3760 drop(table);
3761 if has_deadlines {
3762 self.scope_deadline_added.notify_one();
3763 }
3764 match outcome {
3765 Ok(applied) => {
3766 let counts = ScopeOutcomeCounts::of(&applied.results);
3767 info!(
3768 owner = %owner,
3769 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3770 generation,
3771 records = applied.results.len(),
3772 created = counts.created,
3773 replaced = counts.replaced,
3774 updated = counts.updated,
3775 unchanged = counts.unchanged,
3776 refused = counts.refused,
3777 ended = applied.ended.len(),
3778 tag_changes = applied.tag_changes.len(),
3779 routes_closed = drained.len(),
3780 "scope change accepted"
3781 );
3782 for refused in applied
3788 .results
3789 .iter()
3790 .filter(|result| result.outcome == ScopeRecordOutcome::Refused)
3791 .take(MAX_LOGGED_REFUSED_SCOPE_RECORDS)
3792 {
3793 warn!(
3794 owner = %owner,
3795 generation,
3796 scope_ref = %refused.scope_ref,
3797 scope_epoch = refused.scope_epoch,
3798 code = refused.code.as_deref().unwrap_or(""),
3799 "scope record refused"
3800 );
3801 }
3802 self.close_scope_drained_routes(drained);
3803 let response = if is_apply {
3804 ModuleControlResponseToModule::ScopeApply {
3805 generation,
3806 results: applied.results,
3807 end_results: applied.end_results,
3808 ended: applied.ended,
3809 }
3810 } else {
3811 ModuleControlResponseToModule::ScopeSync {
3812 generation,
3813 results: applied.results,
3814 ended: applied.ended,
3815 }
3816 };
3817 Ok(vec![control_response_body_frame(
3818 &frame,
3819 &response,
3820 "scope change response",
3821 )?])
3822 }
3823 Err(refusal) => {
3824 info!(
3825 owner = %owner,
3826 op = if is_apply { SCOPE_APPLY_OP } else { SCOPE_SYNC_OP },
3827 generation,
3828 code = refusal.code,
3829 "scope change refused"
3830 );
3831 Ok(vec![control_error_frame(
3832 &frame,
3833 refusal.code,
3834 refusal.message,
3835 )?])
3836 }
3837 }
3838 }
3839
3840 fn close_scope_drained_routes(&self, drained: Vec<crate::forwarding::ScopeDrainedRoute>) {
3846 if drained.is_empty() {
3847 return;
3848 }
3849 let mut pushes: BTreeMap<(String, String), (RouteCloseReason, Vec<EndpointRoute>)> =
3850 BTreeMap::new();
3851 let mut goodbyes = Vec::with_capacity(drained.len() * 2);
3852 for route in drained {
3853 warn!(
3854 module_id = %route.module_id,
3855 reason = ?route.reason,
3856 client_connection_id = route.client.connection_id.get(),
3857 route_channel = route.client.channel,
3858 "closing route because its scope changed"
3859 );
3860 pushes
3861 .entry((route.module_id.clone(), format!("{:?}", route.reason)))
3862 .or_insert_with(|| (route.reason, Vec::new()))
3863 .1
3864 .push(EndpointRoute {
3865 goodbye_target: route.client.clone(),
3866 principal: Principal::Unverified,
3867 bound_at: Instant::now(),
3868 draining: false,
3869 drain_reason: None,
3870 });
3871 goodbyes.push(route.module);
3872 goodbyes.push(route.client);
3873 }
3874 for ((module_id, _), (reason, routes)) in pushes {
3875 send_route_control_pushes(
3876 &self.forwarding,
3877 routes,
3878 ClientControlPush::RouteClosed {
3879 module_id,
3880 channels: Vec::new(),
3881 reason,
3882 drained: false,
3883 abandoned: 0,
3884 excluded_subscriptions: 0,
3885 terminal: Some(false),
3886 },
3887 );
3888 }
3889 self.emit_route_goodbyes(goodbyes);
3890 }
3891
3892 fn handle_scope_describe(
3895 &self,
3896 connection_id: ConnectionId,
3897 frame: Frame,
3898 owner: Principal,
3899 scope_ref: String,
3900 ) -> Result<Vec<Frame>, RouterError> {
3901 let registered = self
3902 .registry
3903 .get_module_by_connection(connection_id)
3904 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
3905 let Some(registered) = registered else {
3906 return Ok(vec![control_error_frame(
3907 &frame,
3908 "not_registered",
3909 "scope.describe requires an active module registration owned by this connection",
3910 )?]);
3911 };
3912 let scopes = self
3913 .scopes
3914 .read()
3915 .unwrap_or_else(|poisoned| poisoned.into_inner());
3916 if scopes.is_federated_owner(&owner)
3917 && !crate::scopes::target_supports_federated(registered.manifest.capabilities.as_ref())
3918 {
3919 return Ok(vec![control_error_frame(
3920 &frame,
3921 error_codes::SCOPE_FEDERATED_UNSUPPORTED,
3922 "scope.describe of the federated owner requires scope-federated/v1",
3923 )?]);
3924 }
3925 let mut description = scopes.describe(&owner, &scope_ref);
3926 drop(scopes);
3927 if !registered
3930 .manifest
3931 .capabilities
3932 .as_ref()
3933 .is_some_and(|caps| {
3934 caps.provides
3935 .iter()
3936 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
3937 })
3938 {
3939 if let Some(stamp) = description.stamp.as_mut() {
3940 stamp.attributes.on_behalf_of_agent = None;
3941 }
3942 }
3943 if !registered
3944 .manifest
3945 .capabilities
3946 .as_ref()
3947 .is_some_and(|caps| {
3948 caps.provides
3949 .iter()
3950 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
3951 })
3952 {
3953 if let Some(stamp) = description.stamp.as_mut() {
3954 stamp.attributes.flow_version = None;
3955 stamp.attributes.flow_code_hash = None;
3956 stamp.attributes.flow_install_id = None;
3957 }
3958 }
3959 let owner_configured = match &owner {
3960 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
3965 _ => false,
3966 };
3967 let response = ModuleControlResponseToModule::ScopeDescribe {
3968 status: description.status,
3969 scope_epoch: description.scope_epoch,
3970 daemon_incarnation: self.supervisor.spawn_snapshot().cursor.daemon_incarnation,
3971 owner_synced: description.owner_synced,
3972 owner_configured,
3973 scope: description.stamp,
3974 };
3975 Ok(vec![control_response_body_frame(
3976 &frame,
3977 &response,
3978 "ModuleControlResponseToModule::ScopeDescribe",
3979 )?])
3980 }
3981
3982 fn handle_catalog_update(
3983 &self,
3984 connection_id: ConnectionId,
3985 frame: Frame,
3986 provides: Vec<ProviderRole>,
3987 capabilities: Option<CapabilityDeclarations>,
3988 ready: Option<bool>,
3989 ) -> Result<Vec<Frame>, RouterError> {
3990 self.refresh_capability_requirements();
3991 let Some(registration) = self
3992 .registry
3993 .get_module_by_connection(connection_id)
3994 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
3995 else {
3996 return Ok(vec![control_error_frame(
3997 &frame,
3998 "not_registered",
3999 "catalog.update requires an active module registration owned by this connection",
4000 )?]);
4001 };
4002
4003 if let Some(message) =
4004 catalog_update_frozen_field_message(®istration.manifest, &provides)
4005 {
4006 return Ok(vec![control_error_frame(
4007 &frame,
4008 "catalog_update_frozen_field",
4009 message,
4010 )?]);
4011 }
4012
4013 let mut candidate = registration.manifest.clone();
4014 candidate.provides = provides.clone();
4015 candidate.capabilities = capabilities
4016 .clone()
4017 .or_else(|| registration.manifest.capabilities.clone());
4018 if let Some(rich_ui) = &candidate.rich_ui {
4019 if let Err(err) = rich_ui.validate() {
4020 return Ok(vec![control_error_frame(
4021 &frame,
4022 "invalid_manifest",
4023 err.to_string(),
4024 )?]);
4025 }
4026 }
4027 if let Err(err) = candidate.validate_capability_grammar() {
4028 return Ok(vec![control_error_frame(
4029 &frame,
4030 "invalid_capability_grammar",
4031 err.to_string(),
4032 )?]);
4033 }
4034
4035 let mut conflicts = self
4038 .capability_evaluator
4039 .reserved_hello_refusals(&candidate.module_id, candidate.capabilities.as_ref());
4040 if let Some(conflict) = conflicts.first() {
4041 let message = format!(
4042 "capability '{}' is reserved for module_id '{}'; claimant '{}' was refused",
4043 conflict.capability, conflict.claimants[0], candidate.module_id
4044 );
4045 for conflict in &mut conflicts {
4046 conflict.source = DuplicateClaimSource::CatalogUpdate;
4047 }
4048 log_duplicate_claim_events(conflicts);
4049 return Ok(vec![control_error_frame(
4050 &frame,
4051 "reserved_capability",
4052 message,
4053 )?]);
4054 }
4055
4056 let updated = match self.registry.replace_catalog_for_connection(connection_id, provides, capabilities, ready) {
4057 Err(crate::registry::RegistryError::FederatedCapabilityChanged) => return Ok(vec![control_error_frame(
4058 &frame, error_codes::CATALOG_UPDATE_FROZEN_CAPABILITY,
4059 "scope-federated/v1 is fixed at HELLO; re-register on a new connection to change it",
4060 )?]),
4061 result => result.map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?,
4062 };
4063 if updated.is_none() {
4064 return Ok(vec![control_error_frame(
4065 &frame,
4066 "not_registered",
4067 "catalog.update requires an active module registration owned by this connection",
4068 )?]);
4069 }
4070 if let Ok((_, registrations)) = self.runtime_capability_snapshot() {
4071 log_duplicate_claim_events(
4072 self.capability_evaluator
4073 .duplicate_claims(DuplicateClaimSource::CatalogUpdate, ®istrations),
4074 );
4075 }
4076 if capability_census_trigger(
4077 registration.manifest.capabilities.as_ref(),
4078 updated
4079 .as_ref()
4080 .and_then(|entry| entry.manifest.capabilities.as_ref()),
4081 ) {
4082 self.enforce_capability_denies();
4083 }
4084 self.refresh_capability_requirements();
4085
4086 let response = ModuleControlResponseToModule::CatalogUpdate {};
4087 control_response_body_frame(
4088 &frame,
4089 &response,
4090 "ModuleControlResponseToModule::CatalogUpdate",
4091 )
4092 .map(|frame| vec![frame])
4093 }
4094
4095 pub(crate) fn with_activity_ledger_config(
4096 mut self,
4097 run_dir: &Path,
4098 incarnation: String,
4099 config: crate::activity::ActivityConfig,
4100 ) -> Self {
4101 self.activity = Some(crate::activity::ActivityLedger::open_config(
4102 run_dir.join("activity.jsonl"),
4103 incarnation,
4104 config,
4105 ));
4106 self.supervisor.set_activity(self.activity.clone());
4107 self
4108 }
4109
4110 pub(crate) async fn configure_activity(
4111 &self,
4112 config: crate::activity::ActivityConfig,
4113 ) -> Result<(), &'static str> {
4114 match &self.activity {
4115 Some(ledger) => ledger.configure(config).await,
4116 None => Ok(()),
4117 }
4118 }
4119
4120 fn cancel_activity_subscription(
4121 &self,
4122 ctx: &RouteCtx,
4123 frame: &Frame,
4124 ) -> Result<bool, RouterError> {
4125 let task = self
4126 .activity_streams
4127 .lock()
4128 .unwrap_or_else(|p| p.into_inner())
4129 .remove(&(ctx.connection_id, frame.header.corr));
4130 let Some(task) = task else {
4131 return Ok(false);
4132 };
4133 task.abort();
4134 let end = Frame::build_with_version(
4135 response_version(frame),
4136 FrameType::StreamEnd,
4137 control_flags(),
4138 0,
4139 0,
4140 frame.header.corr,
4141 Vec::new(),
4142 )
4143 .map_err(|error| RouterError::backend(0, frame.header.corr, error.to_string()))?;
4144 let sink = ctx.egress.clone();
4145 tokio::spawn(async move {
4146 let _ = sink.send(end).await;
4147 });
4148 Ok(true)
4149 }
4150
4151 async fn handle_activity_read(
4152 &self,
4153 ctx: &RouteCtx,
4154 frame: Frame,
4155 ) -> Result<Vec<Frame>, RouterError> {
4156 use subc_control::{
4157 ActivityQueryRequest, ActivitySubscribeRequest, ActivitySubscribeResponse,
4158 ACTIVITY_MALFORMED, ACTIVITY_READ_FORBIDDEN, ACTIVITY_UNAVAILABLE,
4159 };
4160 #[derive(Deserialize)]
4161 struct Identity {
4162 #[serde(default)]
4163 consumer_identity: Option<ConsumerIdentity>,
4164 }
4165 let principal = match self
4169 .forwarding
4170 .module_id_for_connection(ctx.connection_id)
4171 .map_err(RouterError::Forwarding)?
4172 {
4173 Some(module_id) => Ok(Principal::Reserved { module_id }),
4174 None => {
4175 let identity: Identity = match serde_json::from_slice(&frame.body) {
4176 Ok(identity) => identity,
4177 Err(_) => {
4178 return Ok(vec![control_error_frame(
4179 &frame,
4180 ACTIVITY_MALFORMED,
4181 "malformed activity read",
4182 )?])
4183 }
4184 };
4185 self.route_open_principal(&frame, identity.consumer_identity)?
4186 }
4187 };
4188 let admitted = matches!(principal, Ok(Principal::Direct))
4189 || matches!(&principal, Ok(Principal::Reserved { module_id }) if module_id == "callosum" || module_id == "ckbus");
4190 if !admitted {
4191 return Ok(vec![control_error_frame(
4192 &frame,
4193 ACTIVITY_READ_FORBIDDEN,
4194 "activity reads require an operator or trusted mirror principal",
4195 )?]);
4196 }
4197 let Some(ledger) = &self.activity else {
4198 return Ok(vec![control_error_frame(
4199 &frame,
4200 ACTIVITY_UNAVAILABLE,
4201 "activity ledger unavailable",
4202 )?]);
4203 };
4204 if serde_json::from_slice::<ControlOpProbe>(&frame.body)
4205 .is_ok_and(|probe| probe.op == ops::ACTIVITY_QUERY)
4206 {
4207 let result = match serde_json::from_slice::<ActivityQueryRequest>(&frame.body) {
4208 Ok(request) => ledger.query(request).await,
4209 Err(_) => Err(ACTIVITY_MALFORMED),
4210 };
4211 return match result {
4212 Ok(reply) => Ok(vec![control_response_body_frame(
4213 &frame,
4214 &reply,
4215 ops::ACTIVITY_QUERY,
4216 )?]),
4217 Err(code) => Ok(vec![control_error_frame(
4218 &frame,
4219 code,
4220 "activity query refused",
4221 )?]),
4222 };
4223 }
4224 let request: ActivitySubscribeRequest = match serde_json::from_slice(&frame.body) {
4225 Ok(request) => request,
4226 Err(_) => {
4227 return Ok(vec![control_error_frame(
4228 &frame,
4229 ACTIVITY_MALFORMED,
4230 "malformed activity subscription",
4231 )?])
4232 }
4233 };
4234 let mut subscription = match ledger.subscribe(request.since).await {
4235 Ok(subscription) => subscription,
4236 Err(code) => {
4237 return Ok(vec![control_error_frame(
4238 &frame,
4239 code,
4240 "activity subscription refused",
4241 )?])
4242 }
4243 };
4244 let key = (ctx.connection_id, frame.header.corr);
4245 let sink = ctx.egress.clone();
4246 let mut streams = self
4247 .activity_streams
4248 .lock()
4249 .unwrap_or_else(|p| p.into_inner());
4250 streams.retain(|_, task| !task.is_finished());
4251 if streams.get(&key).is_some_and(|task| !task.is_finished()) {
4252 return Ok(vec![control_error_frame(
4253 &frame,
4254 "duplicate_subscription",
4255 "activity subscription correlation id already in use",
4256 )?]);
4257 }
4258 let task = tokio::spawn(async move {
4259 loop {
4260 let entry = if let Some(entry) = subscription.replay.pop_front() {
4261 Some(entry)
4262 } else {
4263 subscription.receiver.recv().await
4264 };
4265 let Some(entry) = entry else {
4266 break;
4267 };
4268 let Ok(body) = serde_json::to_vec(&ActivitySubscribeResponse { entry }) else {
4269 return;
4270 };
4271 let Ok(next) = Frame::build_with_version(
4272 response_version(&frame),
4273 FrameType::StreamData,
4274 control_flags(),
4275 0,
4276 0,
4277 frame.header.corr,
4278 body,
4279 ) else {
4280 return;
4281 };
4282 if sink.send(next).await.is_err() {
4283 return;
4284 }
4285 }
4286 if let Some(code) = subscription.terminal_code() {
4289 if let Ok(end) =
4290 control_error_frame(&frame, code, "activity subscriber fell behind")
4291 {
4292 let _ = sink.send(end).await;
4293 }
4294 }
4295 });
4296 streams.insert(key, task.abort_handle());
4297 Ok(Vec::new())
4298 }
4299
4300 async fn handle_activity_record(
4301 &self,
4302 ctx: &RouteCtx,
4303 frame: Frame,
4304 ) -> Result<Vec<Frame>, RouterError> {
4305 use subc_control::{
4306 ActivityRecordRequest, ActivityRecordResponse, ACTIVITY_ACTOR_UNKNOWN,
4307 ACTIVITY_MALFORMED, ACTIVITY_UNAVAILABLE,
4308 };
4309 let result = async {
4310 let module = self
4311 .forwarding
4312 .module_id_for_connection(ctx.connection_id)
4313 .ok()
4314 .flatten()
4315 .ok_or(ACTIVITY_ACTOR_UNKNOWN)?;
4316 let request: ActivityRecordRequest =
4317 serde_json::from_slice(&frame.body).map_err(|_| ACTIVITY_MALFORMED)?;
4318 crate::activity::validate(&module, &request)?;
4319 let actor = self.activity_actor(ctx.connection_id, &module, request.on.as_ref())?;
4320 let ledger = self.activity.as_ref().ok_or(ACTIVITY_UNAVAILABLE)?;
4321 let at_ms = std::time::SystemTime::now()
4322 .duration_since(std::time::UNIX_EPOCH)
4323 .unwrap_or_default()
4324 .as_millis() as u64;
4325 ledger.record(module, request, actor, at_ms).await
4326 }
4327 .await;
4328 match result {
4329 Ok(entry) => Ok(vec![control_response_body_frame(
4330 &frame,
4331 &ActivityRecordResponse { seq: entry.seq },
4332 "activity.record",
4333 )?]),
4334 Err(code) => Ok(vec![control_error_frame(
4335 &frame,
4336 code,
4337 "activity recording refused",
4338 )?]),
4339 }
4340 }
4341
4342 fn activity_actor(
4343 &self,
4344 connection: ConnectionId,
4345 module: &str,
4346 on: Option<&subc_control::ActivityOn>,
4347 ) -> Result<subc_control::ActivityActor, &'static str> {
4348 use crate::forwarding::{DataRoute, DataRouteState};
4349 use subc_control::{ActivityActor, ActivityOn, ACTIVITY_ACTOR_UNKNOWN};
4350 let principal = |id: &str| Principal::Reserved {
4351 module_id: id.to_owned(),
4352 };
4353 let mut actor = ActivityActor {
4354 principal: principal(module),
4355 scope_owner: None,
4356 agent_id: None,
4357 run_id: None,
4358 flow_id: None,
4359 project_root: None,
4360 workdir: None,
4361 project_id: None,
4362 };
4363 let scope = match on {
4364 None => return Ok(actor),
4365 Some(ActivityOn::Scope(scope)) => {
4366 let selector = subc_protocol::scope::ScopeSelector::new(
4367 principal(module),
4368 scope.scope_ref.clone(),
4369 Some(scope.epoch),
4370 );
4371 self.scopes
4372 .read()
4373 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4374 .attribute(&principal(module), module, &selector, true)
4375 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4376 }
4377 Some(ActivityOn::Route(route)) => {
4378 let DataRoute::Module(DataRouteState::Bound(binding)) = self
4379 .forwarding
4380 .lookup_data_route(connection, route.channel, route.epoch)
4381 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4382 else {
4383 return Err(ACTIVITY_ACTOR_UNKNOWN);
4384 };
4385 if binding.module_id != module {
4386 return Err(ACTIVITY_ACTOR_UNKNOWN);
4387 }
4388 actor.principal = binding.principal.clone();
4389 actor.project_root = binding
4390 .project_root
4391 .as_ref()
4392 .map(|root| root.as_path().to_path_buf());
4393 let Some(bound) = &binding.scope else {
4394 return Ok(actor);
4395 };
4396 let owner = principal(&bound.owner);
4397 let selector = subc_protocol::scope::ScopeSelector::new(
4398 owner.clone(),
4399 bound.scope_ref.clone(),
4400 Some(bound.tag.scope_epoch),
4401 );
4402 let admitted = self
4403 .scopes
4404 .read()
4405 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?
4406 .attribute(&owner, module, &selector, true)
4407 .map_err(|_| ACTIVITY_ACTOR_UNKNOWN)?;
4408 if admitted.tag != bound.tag {
4409 return Err(ACTIVITY_ACTOR_UNKNOWN);
4410 }
4411 admitted
4412 }
4413 };
4414 actor.scope_owner = Some(scope.stamp.owner);
4415 actor.agent_id = scope.stamp.attributes.agent_id;
4416 actor.run_id = scope.stamp.attributes.run_id;
4417 actor.flow_id = scope.stamp.attributes.flow_id;
4418 actor.workdir = scope.stamp.workdir.map(std::path::PathBuf::from);
4419 actor.project_id = scope.stamp.project_id;
4420 Ok(actor)
4421 }
4422
4423 fn handle_server_describe(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
4424 self.refresh_capability_requirements();
4425 let mut counters = self.counters.snapshot();
4431 let losses = self.resource_tracker.loss_diagnostics();
4432 if losses.as_object().is_some_and(|losses| !losses.is_empty()) {
4433 if let Some(obj) = counters.as_object_mut() {
4434 obj.insert("agent_command_declaration_loss_by_module".into(), losses);
4435 }
4436 }
4437 if let (Some(ledger), Some(obj)) = (&self.activity, counters.as_object_mut()) {
4438 obj.insert(
4439 "activity_unparseable_lines".into(),
4440 ledger.read_errors().into(),
4441 );
4442 obj.insert(
4443 "activity_lifecycle_append_failures".into(),
4444 ledger.lifecycle_failures().into(),
4445 );
4446 obj.insert(
4447 "activity_started_digest_skips".into(),
4448 ledger.lifecycle_digest_skips().into(),
4449 );
4450 }
4451 if let (Ok((connections_with_routes, max)), Some(obj)) = (
4452 self.forwarding.client_route_concentration(),
4453 counters.as_object_mut(),
4454 ) {
4455 obj.insert(
4456 "client_connections_with_routes".into(),
4457 connections_with_routes.into(),
4458 );
4459 obj.insert("max_routes_on_one_connection".into(), max.into());
4460 }
4461 if let (Some(open_breakers), Some(obj)) = (
4467 self.route_bind_breakers.open_snapshot(),
4468 counters.as_object_mut(),
4469 ) {
4470 obj.insert("route_bind_breakers_open".into(), open_breakers);
4471 }
4472 if let (Some(reason), Some(obj)) = (
4477 self.local_endpoint_unavailable.clone(),
4478 counters.as_object_mut(),
4479 ) {
4480 obj.insert("local_endpoint_unavailable".into(), reason.into());
4481 }
4482 let response = ClientControlResponse::ServerDescribe {
4483 protocol_ver: PROTOCOL_VERSION,
4484 subc_ops: subc_ops(),
4485 capabilities: self.describe_capabilities(),
4486 connected_clients: self.connected_clients.count(),
4487 counters: Some(counters),
4488 build_git_sha: Some(env!("SUBC_BUILD_GIT_SHA").to_string()),
4489 build_lock_digest: Some(env!("SUBC_BUILD_LOCK_DIGEST").to_string()),
4490 capability_requirements: self.capability_requirement_statuses(),
4491 machine_id: self.machine_id.as_ref().map(|id| id.as_str().to_owned()),
4492 config_path: self
4493 .rescan
4494 .as_ref()
4495 .and_then(|context| rescan_checked::reported_config_path(&context.config_path)),
4496 };
4497 Ok(vec![control_response_body_frame(
4498 &frame,
4499 &response,
4500 "ClientControlResponse::ServerDescribe",
4501 )?])
4502 }
4503
4504 fn handle_catalog_list(
4505 &self,
4506 frame: Frame,
4507 module_id: Option<String>,
4508 ) -> Result<Vec<Frame>, RouterError> {
4509 let (generation, modules) = self
4510 .registry
4511 .list_modules_with_nonce_verification()
4512 .map_err(|err| {
4513 RouterError::backend(0, frame.header.corr, format!("registry error: {err}"))
4514 })?;
4515 let entries = modules
4516 .into_iter()
4517 .filter(|(registration, _)| {
4518 module_id
4519 .as_deref()
4520 .map(|wanted| registration.manifest.module_id == wanted)
4521 .unwrap_or(true)
4522 })
4523 .map(|(registration, nonce_verified)| {
4524 let not_ready = self.not_ready_reason(®istration);
4525 let roles = registration.manifest.provides;
4526 CatalogEntry::new(
4527 registration.manifest.module_id,
4528 roles,
4529 registration.control_ops,
4530 )
4531 .with_ready(not_ready.is_none())
4532 .with_display_name(registration.manifest.display_name)
4533 .with_not_ready(not_ready)
4534 .with_module_version(Some(registration.manifest.module_version))
4535 .with_provenance(registration.manifest.provenance)
4536 .with_capabilities(registration.manifest.capabilities)
4537 .with_self_signals(registration.manifest.self_signals)
4538 .with_events(registration.manifest.events)
4539 .with_operation_delivery(registration.manifest.operation_delivery)
4540 .with_operation_payload(registration.manifest.operation_payload)
4541 .with_flow_operations(registration.manifest.flow_operations)
4542 .with_confirm_operations(registration.manifest.confirm_operations)
4543 .with_rich_ui(registration.manifest.rich_ui)
4544 .with_role_declarations(registration.manifest.role_declarations)
4545 .with_nonce_verified(nonce_verified)
4546 })
4547 .collect();
4548 let response = ClientControlResponse::CatalogList {
4549 generation,
4550 modules: entries,
4551 subc_ops: subc_ops(),
4552 };
4553 Ok(vec![control_response_body_frame(
4554 &frame,
4555 &response,
4556 "ClientControlResponse::CatalogList",
4557 )?])
4558 }
4559
4560 fn attest_route_open_consumer(
4565 &self,
4566 consumer_identity: Option<&ConsumerIdentity>,
4567 ) -> Option<Principal> {
4568 let Some(consumer_identity) = consumer_identity else {
4569 return Some(Principal::Direct);
4570 };
4571 self.supervisor
4572 .spawned_consumer_authorized(
4573 &consumer_identity.module_id,
4574 &consumer_identity.launch_nonce,
4575 )
4576 .then(|| Principal::Reserved {
4577 module_id: consumer_identity.module_id.clone(),
4578 })
4579 }
4580
4581 fn route_open_caller_from_frame(&self, frame: &Frame) -> RouteOpenCaller {
4585 let Ok(ClientControlRequest::RouteOpen {
4586 identity,
4587 consumer_identity,
4588 scope,
4589 ..
4590 }) = parse_client_control_request(&frame.body)
4591 else {
4592 return RouteOpenCaller::unknown();
4593 };
4594 let principal = self.attest_route_open_consumer(consumer_identity.as_ref());
4595 RouteOpenCaller::new(principal.as_ref(), scope.as_ref(), &identity)
4596 }
4597
4598 fn route_open_principal(
4599 &self,
4600 frame: &Frame,
4601 consumer_identity: Option<ConsumerIdentity>,
4602 ) -> Result<Result<Principal, Frame>, RouterError> {
4603 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4604 self.route_open_principal_attested(frame, consumer_identity, attested)
4605 }
4606
4607 fn route_open_principal_attested(
4611 &self,
4612 frame: &Frame,
4613 consumer_identity: Option<ConsumerIdentity>,
4614 attested: Option<Principal>,
4615 ) -> Result<Result<Principal, Frame>, RouterError> {
4616 if let Some(principal) = attested {
4617 return Ok(Ok(principal));
4618 }
4619 let Some(consumer_identity) = consumer_identity else {
4620 return Ok(Ok(Principal::Direct));
4621 };
4622
4623 Ok(Err(control_error_frame(
4624 frame,
4625 "bad_consumer_identity",
4626 format!(
4627 "consumer_identity for module_id '{}' did not match a supervised launch nonce",
4628 consumer_identity.module_id
4629 ),
4630 )?))
4631 }
4632
4633 #[allow(clippy::too_many_arguments)]
4640 fn route_open_refusal_frame(
4641 &self,
4642 ctx: &RouteCtx,
4643 caller: &RouteOpenCaller,
4644 frame: &Frame,
4645 module_id: &str,
4646 reason: &'static str,
4647 code: &'static str,
4648 message: impl Into<String>,
4649 ) -> Result<Frame, RouterError> {
4650 self.observe_route_open_refusal(ctx, caller, module_id, reason, code);
4651 control_error_frame(frame, code, message.into())
4652 }
4653
4654 #[allow(clippy::too_many_arguments)]
4663 fn route_bind_rejection(
4664 &self,
4665 ctx: &RouteCtx,
4666 caller: &RouteOpenCaller,
4667 frame: &Frame,
4668 target_module_id: &str,
4669 body: ErrorBody,
4670 daemon_authored: bool,
4671 ) -> Result<Vec<Frame>, RouterError> {
4672 if daemon_authored {
4678 let scope_code = match body.code.as_str() {
4679 error_codes::SCOPE_CHANGED => Some(error_codes::SCOPE_CHANGED),
4680 error_codes::SCOPE_ENDED => Some(error_codes::SCOPE_ENDED),
4681 _ => None,
4682 };
4683 if let Some(code) = scope_code {
4684 self.observe_route_open_refusal(
4685 ctx,
4686 caller,
4687 target_module_id,
4688 "scope_changed_before_commit",
4689 code,
4690 );
4691 return Ok(vec![control_error_body_frame(frame, body)?]);
4692 }
4693 }
4694 self.counters
4695 .increment_route_open_refused("module_rejected");
4696 info!(
4697 target: "control",
4698 code = "module_rejected",
4699 module_code = ?body.code,
4700 module_id = ?target_module_id,
4701 connection_id = ctx.connection_id.get(),
4702 principal = %caller.principal,
4703 scope = %caller.scope,
4704 harness = %caller.harness,
4705 "route.open refused"
4706 );
4707 let refusal = if daemon_authored {
4708 control_error_body_frame(frame, body)?
4709 } else {
4710 control_relayed_error_body_frame(frame, body)?
4711 };
4712 Ok(vec![refusal])
4713 }
4714
4715 #[allow(clippy::too_many_arguments)]
4733 fn route_open_breaker_refusal_frame(
4734 &self,
4735 ctx: &RouteCtx,
4736 caller: &RouteOpenCaller,
4737 frame: &Frame,
4738 module_id: &str,
4739 consecutive_timeouts: u32,
4740 retry_in: Duration,
4741 probe_in_flight: bool,
4742 ) -> Result<Frame, RouterError> {
4743 self.counters
4744 .increment_route_open_refused(crate::observability::ROUTE_OPEN_REFUSED_BREAKER_OPEN);
4745 debug!(
4746 target: "control",
4747 code = "module_timeout",
4748 module_id = ?module_id,
4749 connection_id = ctx.connection_id.get(),
4750 principal = %caller.principal,
4751 scope = %caller.scope,
4752 harness = %caller.harness,
4753 consecutive_timeouts,
4754 retry_in_ms = retry_in.as_millis() as u64,
4755 probe_in_flight,
4756 "route.open refused by open bind-relay breaker"
4757 );
4758 let detail = if probe_in_flight {
4765 "one test route is already being tried; retry once it settles".to_string()
4766 } else {
4767 format!("retrying new routes in {}s", retry_in.as_secs().max(1))
4768 };
4769 control_error_frame(
4770 frame,
4771 "module_timeout",
4772 format!(
4773 "module '{module_id}' is slow to accept new routes ({consecutive_timeouts} \
4774 timed out in a row); {detail}; its established routes are unaffected"
4775 ),
4776 )
4777 }
4778
4779 fn observe_route_open_refusal(
4792 &self,
4793 ctx: &RouteCtx,
4794 caller: &RouteOpenCaller,
4795 module_id: &str,
4796 reason: &'static str,
4797 code: &'static str,
4798 ) {
4799 self.counters.increment_route_open_refused(code);
4800 info!(
4801 target: "control",
4802 code,
4803 reason,
4804 module_id = ?module_id,
4805 connection_id = ctx.connection_id.get(),
4806 principal = %caller.principal,
4807 scope = %caller.scope,
4808 harness = %caller.harness,
4809 "route.open refused"
4810 );
4811 if ROUTE_OPEN_NOT_SERVING_REASONS.contains(&reason) {
4812 self.route_outages.record_not_serving(module_id, reason);
4813 }
4814 }
4815
4816 fn observe_route_open_accept(
4861 &self,
4862 ctx: &RouteCtx,
4863 module_id: &str,
4864 principal: &str,
4865 scope: &str,
4866 ) {
4867 self.route_outages.record_accepted(module_id);
4868 self.counters.increment_route_open_accepted(principal);
4869 info!(
4870 target: "control",
4871 principal,
4872 module_id,
4873 connection_id = ctx.connection_id.get(),
4874 scope = %scope,
4875 "route.open accepted"
4876 );
4877 }
4878
4879 fn supervised_absent_route_open_refusal_frame(
4880 &self,
4881 ctx: &RouteCtx,
4882 caller: &RouteOpenCaller,
4883 frame: &Frame,
4884 module_id: &str,
4885 code: &'static str,
4886 status: &crate::supervise::ModuleStatus,
4887 ) -> Result<Frame, RouterError> {
4888 self.counters.increment_route_open_refused(code);
4889 info!(
4890 target: "control",
4891 code,
4892 reason = "supervised_not_registered",
4893 module_id = ?module_id,
4894 connection_id = ctx.connection_id.get(),
4895 principal = %caller.principal,
4896 scope = %caller.scope,
4897 harness = %caller.harness,
4898 state = %status.state,
4899 enabled = status.enabled,
4900 live = status.live,
4901 "route.open refused"
4902 );
4903 self.route_outages
4907 .record_not_serving(module_id, "supervised_not_registered");
4908 let message = if let Some(pid) = status.waiting_for_previous_pid {
4909 status.waiting_for_previous_detail.clone().unwrap_or_else(|| {
4910 format!("waiting: previous instance (pid {pid}) is still running but could not be verified")
4911 })
4912 } else {
4913 format!(
4914 "module_id '{module_id}' is supervised but not available (state={}, enabled={}, live={})",
4915 status.state, status.enabled, status.live
4916 )
4917 };
4918 control_error_frame(frame, code, message)
4919 }
4920
4921 async fn handle_route_open(
4922 &self,
4923 ctx: &RouteCtx,
4924 frame: Frame,
4925 request: RouteOpenRequest,
4926 timings: &mut RouteOpenTimings,
4927 ) -> Result<Vec<Frame>, RouterError> {
4928 let RouteOpenRequest {
4929 target,
4930 mut identity,
4931 consumer_identity,
4932 consumer_capabilities,
4933 role_versions,
4934 busy_refusals,
4935 admission_facts,
4936 scope,
4937 } = request;
4938 let target_module_id = target_module_id(&target).to_string();
4939 timings.module_id.clone_from(&target_module_id);
4940 if self
4941 .registry
4942 .get_module_by_connection(ctx.connection_id)
4943 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
4944 .is_some()
4945 {
4946 return Ok(vec![control_error_frame(
4947 &frame,
4948 "invalid_request",
4949 "module connections cannot open client routes",
4950 )?]);
4951 }
4952 debug!(
4953 connection_id = ctx.connection_id.get(),
4954 corr = frame.header.corr,
4955 module_id = %target_module_id,
4956 "handling route.open"
4957 );
4958
4959 let attested = self.attest_route_open_consumer(consumer_identity.as_ref());
4964 let mut caller = RouteOpenCaller::new(attested.as_ref(), scope.as_ref(), &identity);
4965
4966 let role_versions = role_versions.filter(|role_versions| !role_versions.is_empty());
4972 if let Some(Err(error)) = role_versions.as_ref().map(validate_role_versions) {
4973 self.observe_route_open_refusal(
4974 ctx,
4975 &caller,
4976 &target_module_id,
4977 "invalid_role_versions",
4978 error_codes::INVALID_REQUEST,
4979 );
4980 return Ok(vec![control_error_body_frame(
4981 &frame,
4982 ErrorBody {
4983 code: error_codes::INVALID_REQUEST.to_string(),
4984 message: error.to_string(),
4985 detail: Some(serde_json::json!({ "field": ROLE_VERSIONS_FIELD })),
4986 },
4987 )?]);
4988 }
4989
4990 if let Some((status, _)) = self.supervisor_status(&target_module_id, frame.header.corr)? {
4991 if status.waiting_for_previous_pid.is_some() {
4992 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
4993 ctx,
4994 &caller,
4995 &frame,
4996 &target_module_id,
4997 "module_warming",
4998 &status,
4999 )?]);
5000 }
5001 }
5002
5003 let Some(registration) = self
5022 .registry
5023 .get_module(&target_module_id)
5024 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5025 else {
5026 if let Some((status, warming)) =
5027 self.supervisor_status(&target_module_id, frame.header.corr)?
5028 {
5029 if status.protocol == ModuleProtocol::None {
5036 return Ok(vec![self.route_open_refusal_frame(
5037 ctx,
5038 &caller,
5039 &frame,
5040 &target_module_id,
5041 "protocol_none",
5042 error_codes::MODULE_NO_PROTOCOL,
5043 format!(
5044 "module_id '{target_module_id}' is declared protocol: none; \
5045 it speaks no subc wire and serves no routes"
5046 ),
5047 )?]);
5048 }
5049 let code = if warming {
5050 "module_warming"
5051 } else {
5052 "target_unavailable"
5053 };
5054 return Ok(vec![self.supervised_absent_route_open_refusal_frame(
5055 ctx,
5056 &caller,
5057 &frame,
5058 &target_module_id,
5059 code,
5060 &status,
5061 )?]);
5062 }
5063 if let Some(removed_ago_ms) =
5064 self.supervisor.removal_tombstone_age_ms(&target_module_id)
5065 {
5066 return Ok(vec![self.route_open_refusal_frame(
5067 ctx,
5068 &caller,
5069 &frame,
5070 &target_module_id,
5071 "removed",
5072 error_codes::MODULE_REMOVED,
5073 format!("module_id '{target_module_id}' was removed {removed_ago_ms} ms ago"),
5074 )?]);
5075 }
5076 return Ok(vec![self.route_open_refusal_frame(
5077 ctx,
5078 &caller,
5079 &frame,
5080 &target_module_id,
5081 "not_registered",
5082 error_codes::UNKNOWN_MODULE,
5083 format!("module_id '{target_module_id}' is not registered"),
5084 )?]);
5085 };
5086
5087 if !registration.ready {
5091 self.counters
5092 .increment_route_open_refused(ROUTE_OPEN_REFUSED_DECLARED_NOT_READY);
5093 info!(
5094 target: "control",
5095 code = error_codes::MODULE_WARMING,
5096 module_id = ?target_module_id,
5097 connection_id = ctx.connection_id.get(),
5098 principal = %caller.principal,
5099 scope = %caller.scope,
5100 harness = %caller.harness,
5101 reason = "declared_not_ready",
5102 "route.open refused"
5103 );
5104 self.route_outages
5107 .record_not_serving(&target_module_id, "declared_not_ready");
5108 return Ok(vec![control_error_body_frame(
5109 &frame,
5110 ErrorBody {
5111 code: error_codes::MODULE_WARMING.to_string(),
5112 message: format!(
5113 "module_id '{target_module_id}' is registered and has declared itself not ready; retry"
5114 ),
5115 detail: Some(serde_json::json!({
5116 "reason": "declared_not_ready"
5117 })),
5118 },
5119 )?]);
5120 }
5121
5122 if let Some(capability) = self.first_unprovided_required_capability(®istration) {
5141 self.counters
5142 .increment_route_open_refused(ROUTE_OPEN_REFUSED_REQUIRED_CAPABILITY_UNPROVIDED);
5143 info!(
5144 target: "control",
5145 code = error_codes::MODULE_WARMING,
5146 module_id = ?target_module_id,
5147 connection_id = ctx.connection_id.get(),
5148 principal = %caller.principal,
5149 scope = %caller.scope,
5150 harness = %caller.harness,
5151 reason = NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5152 capability = %capability,
5153 "route.open refused"
5154 );
5155 return Ok(vec![control_error_body_frame(
5156 &frame,
5157 ErrorBody {
5158 code: error_codes::MODULE_WARMING.to_string(),
5159 message: format!(
5160 "module_id '{target_module_id}' requires capability '{capability}', \
5161 which no registered module provides; retry"
5162 ),
5163 detail: Some(serde_json::json!({
5164 "reason": NotReadyReason::REQUIRED_CAPABILITY_UNPROVIDED,
5165 "capability": capability,
5166 })),
5167 },
5168 )?]);
5169 }
5170
5171 if !target_has_required_role(&target, ®istration.manifest.provides) {
5172 return Ok(vec![self.route_open_refusal_frame(
5173 ctx,
5174 &caller,
5175 &frame,
5176 &target_module_id,
5177 "role_not_provided",
5178 "target_unavailable",
5179 format!("module_id '{target_module_id}' does not provide the requested target"),
5180 )?]);
5181 }
5182
5183 if registration.state != ChannelState::Active {
5184 return Ok(vec![self.route_open_refusal_frame(
5185 ctx,
5186 &caller,
5187 &frame,
5188 &target_module_id,
5189 "registration_not_active",
5190 "target_unavailable",
5191 format!("module_id '{target_module_id}' is not active"),
5192 )?]);
5193 }
5194
5195 if self
5196 .forwarding
5197 .module_is_draining(&target_module_id)
5198 .map_err(RouterError::Forwarding)?
5199 {
5200 return Ok(vec![self.route_open_refusal_frame(
5201 ctx,
5202 &caller,
5203 &frame,
5204 &target_module_id,
5205 "reloading",
5206 "module_reloading",
5207 format!("module_id '{target_module_id}' is reloading"),
5208 )?]);
5209 }
5210
5211 if let Some(process_liveness) = self.process_liveness.as_ref().filter(|process_liveness| {
5212 process_liveness.process_live(&target_module_id) == Some(false)
5213 }) {
5214 if process_liveness.process_replacing(&target_module_id) {
5221 return Ok(vec![self.route_open_refusal_frame(
5222 ctx,
5223 &caller,
5224 &frame,
5225 &target_module_id,
5226 "reloading",
5227 "module_reloading",
5228 format!("module_id '{target_module_id}' is reloading"),
5229 )?]);
5230 }
5231 return Ok(vec![self.route_open_refusal_frame(
5232 ctx,
5233 &caller,
5234 &frame,
5235 &target_module_id,
5236 "supervisor_not_live",
5237 "target_unavailable",
5238 format!("module_id '{target_module_id}' is not live"),
5239 )?]);
5240 }
5241
5242 if !self
5243 .forwarding
5244 .has_live_module_connection(&target_module_id)
5245 .map_err(RouterError::Forwarding)?
5246 {
5247 return Ok(vec![self.route_open_refusal_frame(
5248 ctx,
5249 &caller,
5250 &frame,
5251 &target_module_id,
5252 "no_forwarding_connection",
5253 "target_unavailable",
5254 format!("module_id '{target_module_id}' has no live forwarding connection"),
5255 )?]);
5256 }
5257
5258 if let Some(error) =
5259 self.guard_module_control_op(&frame, &target_module_id, "route.bind")?
5260 {
5261 self.observe_route_open_refusal(
5262 ctx,
5263 &caller,
5264 &target_module_id,
5265 "op_not_allowed",
5266 "op_not_allowed",
5267 );
5268 return Ok(vec![error]);
5269 }
5270
5271 let principal =
5272 match self.route_open_principal_attested(&frame, consumer_identity, attested)? {
5273 Ok(principal) => principal,
5274 Err(error) => {
5275 self.observe_route_open_refusal(
5276 ctx,
5277 &caller,
5278 &target_module_id,
5279 "bad_consumer_identity",
5280 "bad_consumer_identity",
5281 );
5282 return Ok(vec![error]);
5283 }
5284 };
5285 timings.principal = principal_label(&principal);
5286
5287 let build_class = identity.build_class.take().filter(|build_class| {
5290 if matches!(&principal, Principal::Reserved { module_id } if module_id == "callosum") {
5291 build_class == subc_protocol::BUILD_CLASS_PRODUCTION
5292 } else {
5293 if self
5294 .build_class_warned
5295 .lock()
5296 .unwrap_or_else(|p| p.into_inner())
5297 .insert(ctx.connection_id)
5298 {
5299 warn!(connection_id = ctx.connection_id.get(), principal = %timings.principal,
5300 "ignoring build_class from a non-Callosum opener");
5301 }
5302 false
5303 }
5304 });
5305
5306 if let Principal::Reserved {
5310 module_id: opening_module_id,
5311 } = &principal
5312 {
5313 if let Some(opening_registration) = self
5314 .registry
5315 .get_module(opening_module_id)
5316 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
5317 {
5318 if let Some(capability) =
5319 denied_capability(&opening_registration.manifest, ®istration.manifest)
5320 {
5321 warn!(
5322 opening_module_id,
5323 target_module_id,
5324 capability,
5325 "refusing route.open because an attested capability deny edge matches"
5326 );
5327 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5328 &frame,
5329 &target_module_id,
5330 "capability_deny_edge",
5331 "capability_forbidden",
5332 format!(
5333 "module_id '{opening_module_id}' must never reach capability '{capability}' provided by '{target_module_id}'"
5334 ),
5335 )?]);
5336 }
5337 }
5338 }
5339
5340 if admission_facts.is_some() {
5341 let carrier_matches = matches!(
5342 &principal,
5343 Principal::Reserved { module_id }
5344 if self.admission_facts_carrier_module_id.as_deref() == Some(module_id)
5345 );
5346 if !carrier_matches {
5347 return Ok(vec![self.route_open_refusal_frame(
5348 ctx,
5349 &caller,
5350 &frame,
5351 &target_module_id,
5352 "admission_facts_carrier_not_permitted",
5353 "admission_facts_not_permitted",
5354 "admission facts may only be carried by the configured reserved module",
5355 )?]);
5356 }
5357
5358 let target_allowed = self
5359 .admission_facts_targets
5360 .as_ref()
5361 .is_some_and(|targets| targets.iter().any(|id| id == &target_module_id));
5362 if !target_allowed {
5363 return Ok(vec![self.route_open_refusal_frame(ctx, &caller,
5364 &frame,
5365 &target_module_id,
5366 "admission_facts_target_not_listed",
5367 "admission_facts_target_not_allowed",
5368 format!(
5369 "admission facts are not permitted for target module_id '{target_module_id}'"
5370 ),
5371 )?]);
5372 }
5373
5374 }
5378
5379 let (bound_scope, mut scope_stamp) = match &scope {
5385 None => (None, None),
5386 Some(selector) => {
5387 let owner_configured = match &selector.owner {
5388 Principal::Reserved { module_id } => self.supervisor.is_configured(module_id),
5394 _ => false,
5395 };
5396 let mut scopes = self
5397 .scopes
5398 .write()
5399 .unwrap_or_else(|poisoned| poisoned.into_inner());
5400 let admitted = scopes
5401 .admit(&principal, &target_module_id, selector, owner_configured)
5402 .and_then(|admission| {
5403 crate::scopes::check_target_flow_support(
5404 &admission.stamp,
5405 &target_module_id,
5406 registration.manifest.capabilities.as_ref(),
5407 )?;
5408 crate::scopes::check_target_agent_run_support(
5409 &admission.stamp,
5410 &target_module_id,
5411 registration.manifest.capabilities.as_ref(),
5412 )?;
5413 let bound = scopes.reserve_bind(
5414 &admission,
5415 &principal,
5416 ctx.connection_id,
5417 selector.hold_presence,
5418 );
5419 Ok((admission, bound))
5420 });
5421 match admitted {
5422 Ok((admission, bound)) => (Some(bound), Some(admission.stamp)),
5423 Err(refusal) => {
5424 return Ok(vec![self.route_open_refusal_frame(
5425 ctx,
5426 &caller,
5427 &frame,
5428 &target_module_id,
5429 refusal.code,
5430 refusal.code,
5431 refusal.message,
5432 )?]);
5433 }
5434 }
5435 }
5436 };
5437 if let Some(stamp) = &scope_stamp {
5440 caller.scope = route_open_scope_label(stamp);
5441 }
5442
5443 timings.enter(RouteOpenPhase::ProjectRoot);
5451 let legacy_project = match crate::bind_compat::LegacyProject::from_open(&frame.body) {
5452 Ok(project) => project,
5453 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
5454 };
5455 timings.enter(RouteOpenPhase::Admission);
5456 let project_root = match legacy_project.effective_root(scope_stamp.as_ref()) {
5457 Ok(root) => root,
5458 Err(err) => {
5459 return Ok(vec![control_error_frame(
5460 &frame,
5461 "invalid_project_root",
5462 err.to_string(),
5463 )?])
5464 }
5465 };
5466 let legacy_bind_project = legacy_project.for_target(scope_stamp.as_ref());
5472
5473 let _concurrency_guard = match self
5484 .route_bind_concurrency
5485 .try_admit(&target_module_id, MAX_PENDING_ROUTE_BINDS_PER_TARGET)
5486 {
5487 Ok(guard) => guard,
5488 Err(in_flight) => {
5489 return Ok(vec![self.route_open_target_capacity_refusal(
5490 ctx,
5491 &caller,
5492 &frame,
5493 &target_module_id,
5494 in_flight,
5495 )?]);
5496 }
5497 };
5498
5499 let mut breaker = match self.route_bind_breakers.admit(&target_module_id) {
5502 RouteBindAdmission::Admitted { guard, probe } => {
5503 if probe {
5504 info!(
5505 module_id = %target_module_id,
5506 connection_id = ctx.connection_id.get(),
5507 "route.bind breaker half-open: admitting one probe"
5508 );
5509 }
5510 guard
5511 }
5512 RouteBindAdmission::Refused {
5513 consecutive_timeouts,
5514 retry_in,
5515 probe_in_flight,
5516 } => {
5517 return Ok(vec![self.route_open_breaker_refusal_frame(
5518 ctx,
5519 &caller,
5520 &frame,
5521 &target_module_id,
5522 consecutive_timeouts,
5523 retry_in,
5524 probe_in_flight,
5525 )?]);
5526 }
5527 };
5528
5529 let route_bind_relay_timeout = self.route_bind_relay_timeout_for(&target_module_id);
5534 let relay_deadline = Instant::now() + route_bind_relay_timeout;
5535 let operator_app = self
5536 .operator_apps
5537 .lock()
5538 .unwrap_or_else(|p| p.into_inner())
5539 .get(&ctx.connection_id)
5540 .cloned();
5541 timings.enter(RouteOpenPhase::EgressReservation);
5542 let pending = match self
5543 .forwarding
5544 .begin_route_bind_relay_for(
5545 ctx.connection_id,
5546 ctx.egress.clone(),
5547 response_version(&frame),
5548 frame.header.corr,
5549 &target_module_id,
5550 principal.clone(),
5551 bound_scope,
5552 project_root,
5553 identity.harness.clone(),
5554 build_class,
5555 operator_app.clone(),
5556 relay_deadline,
5557 busy_refusals == Some(true),
5558 )
5559 .await
5560 {
5561 Ok(pending) => pending,
5562 Err(err) => {
5563 return Ok(vec![self.route_open_refusal_frame(
5564 ctx,
5565 &caller,
5566 &frame,
5567 &target_module_id,
5568 "relay_reservation_failed",
5569 forwarding_error_code(&err),
5570 err.to_string(),
5571 )?])
5572 }
5573 };
5574 timings.enter(RouteOpenPhase::Admission);
5575 let crate::forwarding::PendingRouteBindRelay {
5576 endpoint,
5577 module_sink,
5578 negotiated_ver,
5579 client_channel,
5580 client_epoch,
5581 module_channel,
5582 module_epoch,
5583 corr: relay_corr,
5584 receiver,
5585 } = pending;
5586 let mut reservation =
5587 RouteBindReservationGuard::new(Arc::clone(&self.forwarding), endpoint, relay_corr);
5588
5589 let relay_registration = self
5594 .registry
5595 .get_module_by_connection(endpoint.connection_id)
5596 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
5597 let relay_capabilities = relay_registration
5598 .as_ref()
5599 .and_then(|registration| registration.manifest.capabilities.as_ref());
5600 if let Some(selector) = &scope {
5601 if let Err(refusal) = self
5604 .scopes
5605 .read()
5606 .unwrap_or_else(|poisoned| poisoned.into_inner())
5607 .check_bind_proof(&principal, selector)
5608 {
5609 reservation.release_and_disarm();
5610 return Ok(vec![self.route_open_refusal_frame(
5611 ctx,
5612 &caller,
5613 &frame,
5614 &target_module_id,
5615 refusal.code,
5616 refusal.code,
5617 refusal.message,
5618 )?]);
5619 }
5620 }
5621 if let Some(stamp) = scope_stamp.as_ref() {
5622 if let Err(refusal) = crate::scopes::check_target_flow_support(
5623 stamp,
5624 &target_module_id,
5625 relay_capabilities,
5626 )
5627 .and_then(|()| {
5628 crate::scopes::check_target_agent_run_support(
5629 stamp,
5630 &target_module_id,
5631 relay_capabilities,
5632 )
5633 }) {
5634 reservation.release_and_disarm();
5635 return Ok(vec![self.route_open_refusal_frame(
5636 ctx,
5637 &caller,
5638 &frame,
5639 &target_module_id,
5640 refusal.code,
5641 refusal.code,
5642 refusal.message,
5643 )?]);
5644 }
5645 }
5646
5647 if scope_stamp
5650 .as_ref()
5651 .is_some_and(|stamp| stamp.kind == subc_protocol::scope::ScopeKind::Federated)
5652 && !crate::scopes::target_supports_federated(relay_capabilities)
5653 {
5654 scope_stamp = None;
5656 }
5657 if !relay_capabilities.is_some_and(|caps| {
5658 caps.provides
5659 .iter()
5660 .any(|cap| cap == subc_protocol::scope::SCOPE_WORKDIR_CAPABILITY)
5661 }) {
5662 if let Some(stamp) = scope_stamp.as_mut() {
5663 stamp.workdir = None;
5664 stamp.project_id = None;
5665 }
5666 }
5667 let worker_run = scope_stamp
5672 .as_ref()
5673 .is_some_and(crate::scopes::is_worker_run)
5674 && relay_capabilities.is_some_and(|caps| {
5675 caps.provides
5676 .iter()
5677 .any(|cap| cap == subc_protocol::scope::WORKER_RUNS_CAPABILITY)
5678 });
5679 if !worker_run
5680 && !relay_capabilities.is_some_and(|caps| {
5681 caps.provides
5682 .iter()
5683 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY)
5684 })
5685 {
5686 if let Some(stamp) = scope_stamp.as_mut() {
5687 stamp.attributes.on_behalf_of_agent = None;
5688 }
5689 }
5690 if !relay_capabilities.is_some_and(|caps| {
5691 caps.provides
5692 .iter()
5693 .any(|cap| cap == subc_protocol::scope::FLOW_IDENTITY_SCOPES_CAPABILITY)
5694 }) {
5695 if let Some(stamp) = scope_stamp.as_mut() {
5696 stamp.attributes.flow_version = None;
5697 stamp.attributes.flow_code_hash = None;
5698 stamp.attributes.flow_install_id = None;
5699 }
5700 }
5701 let call_mode = if worker_run
5702 || relay_capabilities
5703 .is_some_and(|caps| caps.provides.iter().any(|cap| cap == CALL_MODES_CAPABILITY))
5704 {
5705 Some(derive_call_mode(scope_stamp.as_ref()))
5706 } else {
5707 if let Some(stamp) = scope_stamp.as_mut() {
5708 stamp.expires_at_ms = None;
5709 }
5710 None
5711 };
5712
5713 debug!(
5714 connection_id = ctx.connection_id.get(),
5715 client_channel,
5716 client_epoch,
5717 module_channel,
5718 module_epoch,
5719 "reserved route handle pair"
5720 );
5721 let route_principal_label = principal_label(&principal);
5724 let relay = ModuleControlRequest::RouteBind {
5725 route_channel: module_channel,
5726 epoch: module_epoch,
5727 target,
5728 identity,
5729 principal: Some(principal),
5730 consumer_capabilities,
5731 role_versions,
5732 admission_facts,
5733 scope: scope_stamp,
5734 call_mode,
5735 };
5736 let peer = RouteBindPeer::new()
5737 .with_observed_peer(
5738 relay_capabilities
5739 .is_some_and(|caps| {
5740 caps.provides
5741 .iter()
5742 .any(|cap| cap == OBSERVED_PEER_CAPABILITY)
5743 })
5744 .then(|| {
5745 self.observed_peers
5746 .lock()
5747 .unwrap_or_else(|p| p.into_inner())
5748 .get(&ctx.connection_id)
5749 .cloned()
5750 .unwrap_or_else(ObservedPeer::unobserved)
5751 }),
5752 )
5753 .with_operator_app(
5754 self.connection_operator_app(operator_app.as_ref())
5755 .filter(|_| {
5756 relay_capabilities.is_some_and(|caps| {
5757 caps.provides
5758 .iter()
5759 .any(|cap| cap == OPERATOR_APP_CAPABILITY)
5760 })
5761 }),
5762 );
5763 let relay_body = crate::bind_compat::encode_bind(
5764 &relay,
5765 &peer,
5766 &legacy_bind_project,
5767 relay_capabilities,
5768 )
5769 .map_err(|err| {
5770 RouterError::backend(
5771 0,
5772 frame.header.corr,
5773 format!("failed to encode route.bind request: {err}"),
5774 )
5775 })?;
5776 let relay_frame = Frame::build_with_version(
5777 negotiated_ver,
5778 FrameType::Request,
5779 control_flags(),
5780 0,
5781 0,
5782 relay_corr,
5783 relay_body,
5784 )
5785 .map_err(RouterError::FrameBuild)?;
5786
5787 timings.enter(RouteOpenPhase::BindSend);
5788 let sent = module_sink.send(relay_frame).await;
5789 timings.enter(RouteOpenPhase::Admission);
5790 if let Err(err) = sent {
5791 reservation.release_and_disarm();
5792 return Ok(vec![self.route_open_refusal_frame(
5793 ctx,
5794 &caller,
5795 &frame,
5796 &target_module_id,
5797 "relay_send_failed",
5798 "target_unavailable",
5799 err.to_string(),
5800 )?]);
5801 }
5802
5803 #[cfg(test)]
5804 if let Some(hold) = &self.route_bind_relay_sent_hold {
5805 hold.wait().await;
5809 hold.wait().await;
5810 }
5811
5812 if self
5817 .forwarding
5818 .mark_route_bind_relay_enqueued(endpoint, relay_corr)
5819 .map_err(RouterError::Forwarding)?
5820 == crate::forwarding::RouteBindRelayMark::ReleasedWithoutReply
5821 {
5822 self.send_abandoned_route_bind_goodbye(
5823 &module_sink,
5824 negotiated_ver,
5825 module_channel,
5826 module_epoch,
5827 );
5828 }
5829
5830 timings.enter(RouteOpenPhase::BindAckWait);
5831 let outcome = timeout_at(relay_deadline, receiver).await;
5832 timings.enter(RouteOpenPhase::Admission);
5833 match outcome {
5834 Ok(Ok(RouteBindRelayOutcome::Accepted)) => {
5835 reservation.disarm();
5836 if breaker.record_accepted() {
5837 info!(
5838 module_id = %target_module_id,
5839 "route.bind breaker closed: the probe was accepted"
5840 );
5841 }
5842 self.observe_route_open_accept(
5843 ctx,
5844 &target_module_id,
5845 &route_principal_label,
5846 &caller.scope,
5847 );
5848 Ok(Vec::new())
5849 }
5850 Ok(Ok(RouteBindRelayOutcome::Rejected(body))) => {
5851 reservation.release_and_disarm();
5852 breaker.record_inconclusive();
5856 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, false)
5857 }
5858 Ok(Ok(RouteBindRelayOutcome::DaemonRejected(body))) => {
5859 reservation.release_and_disarm();
5860 breaker.record_inconclusive();
5861 self.route_bind_rejection(ctx, &caller, &frame, &target_module_id, body, true)
5862 }
5863 Ok(Ok(RouteBindRelayOutcome::ModuleGone(message))) => {
5864 reservation.release_and_disarm();
5865 breaker.record_inconclusive();
5866 tracing::warn!(
5873 module_id = %target_module_id,
5874 "route.bind relay abandoned: {message}"
5875 );
5876 Ok(vec![self.route_open_refusal_frame(
5877 ctx,
5878 &caller,
5879 &frame,
5880 &target_module_id,
5881 "relay_abandoned",
5882 "target_unavailable",
5883 message,
5884 )?])
5885 }
5886 Ok(Err(_)) => {
5887 reservation.release_and_disarm();
5888 breaker.record_inconclusive();
5889 Ok(vec![self.route_open_refusal_frame(
5890 ctx,
5891 &caller,
5892 &frame,
5893 &target_module_id,
5894 "relay_waiter_canceled",
5895 "target_unavailable",
5896 "route.bind relay waiter was canceled before the module responded",
5897 )?])
5898 }
5899 Err(_) => {
5900 reservation.release_and_disarm();
5901 if let Some(opened) = breaker.record_timeout(
5905 self.route_bind_breaker_threshold,
5906 self.route_bind_breaker_cooldown,
5907 ) {
5908 warn!(
5909 module_id = %target_module_id,
5910 consecutive_timeouts = opened.consecutive_timeouts,
5911 cooldown_ms = self.route_bind_breaker_cooldown.as_millis() as u64,
5912 reopened_after_probe = opened.reopened_after_probe,
5913 "route.bind breaker open: refusing route.open for this module without relaying until one probe says it recovered"
5914 );
5915 }
5916 tracing::warn!(
5924 module_id = %target_module_id,
5925 timeout_ms = route_bind_relay_timeout.as_millis() as u64,
5926 "route.bind relay timed out: module did not ack within budget"
5927 );
5928 Ok(vec![self.route_open_refusal_frame(
5929 ctx,
5930 &caller,
5931 &frame,
5932 &target_module_id,
5933 "relay_timed_out",
5934 "module_timeout",
5935 format!(
5936 "module_id '{target_module_id}' did not answer route.bind within {:?}",
5937 route_bind_relay_timeout
5938 ),
5939 )?])
5940 }
5941 }
5942 }
5943
5944 fn handle_supervisor_spawn_snapshot(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
5945 let response = ClientControlResponse::SupervisorSpawnSnapshot {
5946 snapshot: self.supervisor.spawn_snapshot(),
5947 };
5948 Ok(vec![control_response_body_frame(
5949 &frame,
5950 &response,
5951 "ClientControlResponse::SupervisorSpawnSnapshot",
5952 )?])
5953 }
5954
5955 fn handle_supervisor_spawn_subscribe(
5956 &self,
5957 ctx: &RouteCtx,
5958 frame: Frame,
5959 since: Option<SpawnCursor>,
5960 ) -> Result<Vec<Frame>, RouterError> {
5961 match self.supervisor.subscribe_spawns(
5962 ctx.connection_id,
5963 frame.header.corr,
5964 response_version(&frame),
5965 since,
5966 ctx.egress.clone(),
5967 ) {
5968 Ok(()) => Ok(Vec::new()),
5969 Err(SpawnSubscribeRefusal::ForeignIncarnation { current }) => {
5970 Ok(vec![control_error_body_frame(
5971 &frame,
5972 ErrorBody {
5973 code: "spawn_cursor_incarnation_mismatch".to_string(),
5974 message: "spawn cursor belongs to a different daemon incarnation"
5975 .to_string(),
5976 detail: Some(serde_json::json!({
5977 "current_daemon_incarnation": current
5978 })),
5979 },
5980 )?])
5981 }
5982 Err(SpawnSubscribeRefusal::TooOld { oldest }) => Ok(vec![control_error_body_frame(
5983 &frame,
5984 ErrorBody {
5985 code: "spawn_cursor_too_old".to_string(),
5986 message: "spawn cursor predates the retained event ring".to_string(),
5987 detail: Some(serde_json::json!({
5988 "oldest_retained_cursor": oldest
5989 })),
5990 },
5991 )?]),
5992 Err(SpawnSubscribeRefusal::Frame(error)) => Err(RouterError::backend(
5993 0,
5994 frame.header.corr,
5995 format!("failed to open supervisor spawn subscription: {error}"),
5996 )),
5997 }
5998 }
5999
6000 async fn handle_supervisor_list(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6001 let handler = self.clone();
6004 let runtime = tokio::runtime::Handle::current();
6005 let subscriber = tracing::dispatcher::get_default(Clone::clone);
6008 let corr = frame.header.corr;
6009 tokio::task::spawn_blocking(move || {
6010 runtime.block_on(
6011 list_diagnostics::scope(handler.handle_supervisor_list_inner(frame))
6012 .with_subscriber(subscriber),
6013 )
6014 })
6015 .await
6016 .map_err(|error| RouterError::backend(0, corr, error.to_string()))?
6017 }
6018
6019 async fn handle_supervisor_list_inner(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6020 let generation = self
6021 .registry
6022 .generation()
6023 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6024 let supervised = self.supervisor.list();
6025 let mut statuses = BTreeMap::new();
6026 let mut not_running = BTreeSet::new();
6027 for module in &supervised {
6028 let status = module.status_for_control("list").map_err(|err| {
6029 RouterError::backend(
6030 0,
6031 frame.header.corr,
6032 format!("failed to read supervisor status: {err}"),
6033 )
6034 })?;
6035 if !status.enabled || !status.process_alive {
6036 not_running.insert(module.module_id().to_owned());
6037 }
6038 statuses.insert(module.module_id().to_owned(), status);
6039 }
6040 let mut process_resources = BTreeMap::new();
6043 for module in &supervised {
6044 process_resources.insert(
6045 module.module_id().to_owned(),
6046 crate::subtree_resources::read_process(module.clone()).await,
6047 );
6048 }
6049 let mut image_readings = BTreeMap::new();
6052 for module in &supervised {
6053 image_readings.insert(
6054 module.module_id().to_owned(),
6055 module.running_image_agreement().await,
6056 );
6057 }
6058 let readings = self
6059 .resource_tracker
6060 .read(supervised.clone(), false, "list")
6061 .await;
6062 let mut modules = Vec::new();
6063 for module in supervised {
6064 let status = statuses
6065 .remove(module.module_id())
6066 .expect("each supervised module has a status snapshot");
6067 let (configured, _) = module.configuration().map_err(|err| {
6068 RouterError::backend(
6069 0,
6070 frame.header.corr,
6071 format!("failed to read module configuration: {err}"),
6072 )
6073 })?;
6074 let next_retry_at_ms = module.next_retry_at_ms().map_err(|err| {
6075 RouterError::backend(
6076 0,
6077 frame.header.corr,
6078 format!("failed to read retry time: {err}"),
6079 )
6080 })?;
6081 let image = image_readings
6082 .remove(module.module_id())
6083 .expect("each supervised module has an image snapshot");
6084 let resources = process_resources.remove(module.module_id());
6085 let mut verdict =
6086 reload_verdict(&configured.program, status.spawned_from.as_deref(), image);
6087 verdict.spec = Some(launch_spec_agreement(
6088 &configured,
6089 status.process_alive,
6090 status.launch_spec_digest.as_ref(),
6091 ));
6092 let pending_reload = Some(verdict);
6093 let configured_enabled = self
6094 .capability_evaluator
6095 .configured_enabled(&status.module_id);
6096 modules.push(
6099 SupervisorEntry::new(
6100 status.module_id,
6101 status.state.to_string(),
6102 status.enabled,
6103 status.live,
6104 status.health.status,
6105 )
6106 .with_configured_enabled(configured_enabled)
6107 .with_next_retry_at_ms(next_retry_at_ms)
6108 .with_launch_nonce_env(Some(!cfg!(unix)))
6109 .with_launch_nonce_source(status.launch_nonce_source)
6110 .with_waiting_for_previous_pid(status.waiting_for_previous_pid)
6111 .with_protocol(status.protocol)
6112 .with_pending_reload(pending_reload)
6113 .with_last_probe_ms(status.health.last_probe_ms)
6114 .with_last_exit_code(status.last_exit.as_ref().and_then(|e| e.code))
6115 .with_last_exit_signal(status.last_exit.as_ref().and_then(|e| e.signal))
6116 .with_last_exit_ms(status.last_exit.as_ref().map(|e| e.at_ms))
6117 .with_last_exit_kind(status.last_exit.as_ref().map(|e| e.kind.into()))
6118 .with_restart_count(Some(status.restart_count))
6119 .with_max_restarts(Some(status.max_restarts))
6120 .with_lifetime_restarts(Some(status.lifetime_restarts))
6121 .with_spawn_generation(Some(status.spawn_generation))
6122 .with_restart_window_secs(Some(status.restart_window.as_secs()))
6123 .with_drain_timeout_ms(Some(status.drain_timeout.as_millis() as u64))
6124 .with_restart_backoff_ms(Some(status.restart_backoff.as_millis() as u64))
6125 .with_restart_max_backoff_ms(Some(status.restart_max_backoff.as_millis() as u64))
6126 .with_resources(resources),
6127 );
6128 }
6129 for module in &mut modules {
6130 if let Some((tree, hour)) = readings.modules.get(&module.module_id) {
6131 module.tree = tree.clone();
6132 module.cpu_last_hour = hour.clone();
6133 }
6134 if module.tree.is_none() {
6135 module.tree_unavailable = Some(
6136 if not_running.contains(&module.module_id) {
6137 "not_running"
6138 } else if readings.modules.contains_key(&module.module_id) {
6139 "tree_unavailable"
6140 } else {
6141 "not_yet_measured"
6142 }
6143 .to_owned(),
6144 );
6145 }
6146 }
6147 let response = ClientControlResponse::SupervisorList {
6148 generation,
6149 modules,
6150 daemon: readings.daemon,
6151 agent_commands: readings.agent_commands,
6152 };
6153 Ok(vec![control_response_body_frame(
6154 &frame,
6155 &response,
6156 "ClientControlResponse::SupervisorList",
6157 )?])
6158 }
6159
6160 fn handle_supervisor_stderr_tail(
6161 &self,
6162 frame: Frame,
6163 module_id: String,
6164 max_lines: Option<u32>,
6165 max_bytes: Option<u32>,
6166 ) -> Result<Vec<Frame>, RouterError> {
6167 let Some(module) = self.supervisor.get(&module_id) else {
6168 return Ok(vec![control_error_frame(
6169 &frame,
6170 "unknown_module",
6171 format!("module_id '{module_id}' is not supervised"),
6172 )?]);
6173 };
6174
6175 let snapshot = module.stderr_tail(
6176 max_lines.map(|value| value as usize),
6177 max_bytes.map(|value| value as usize),
6178 );
6179
6180 let response = ClientControlResponse::SupervisorStderrTail {
6181 module_id,
6182 tail: StderrTail {
6183 capture: match snapshot.capture {
6184 CaptureState::Captured => StderrCaptureState::Captured,
6185 CaptureState::Incomplete { reason } => {
6186 StderrCaptureState::Incomplete { reason }
6187 }
6188 CaptureState::NotCaptured { reason } => {
6189 StderrCaptureState::NotCaptured { reason }
6190 }
6191 },
6192 entries: snapshot
6193 .entries
6194 .into_iter()
6195 .map(|entry| match entry {
6196 TailEntry::Line {
6197 text,
6198 truncated,
6199 at_ms,
6200 } => StderrTailEntry::Line {
6201 text,
6202 truncated,
6203 at_ms,
6204 },
6205 TailEntry::ProcessStart => StderrTailEntry::ProcessStart,
6206 })
6207 .collect(),
6208 dropped_lines: snapshot.dropped_lines,
6209 },
6210 };
6211 Ok(vec![control_response_body_frame(
6212 &frame,
6213 &response,
6214 "ClientControlResponse::SupervisorStderrTail",
6215 )?])
6216 }
6217
6218 async fn observe_control_wait<F: std::future::Future>(
6219 &self,
6220 ctx: &RouteCtx,
6221 target: Option<&str>,
6222 kind: &'static str,
6223 future: F,
6224 ) -> F::Output {
6225 self.counters
6226 .observe_reader_wait(
6227 || {
6228 let module = self
6229 .forwarding
6230 .module_id_for_connection(ctx.connection_id)
6231 .ok()
6232 .flatten();
6233 let principal = if module.is_some() { "module" } else { "direct" };
6234 (
6235 self.resolve_reader_wait_target(kind, target)
6236 .or(module)
6237 .unwrap_or_else(|| crate::observability::CONTROL_WAIT_BUCKET.into()),
6238 principal.into(),
6239 )
6240 },
6241 kind,
6242 future,
6243 )
6244 .await
6245 }
6246
6247 pub(crate) fn resolve_reader_wait_target(
6250 &self,
6251 op: &str,
6252 selector: Option<&str>,
6253 ) -> Option<String> {
6254 let selector = selector?;
6255 match op {
6256 ops::SUPERVISOR_TERMINALS | ops::SUPERVISOR_PROVENANCE => self
6257 .supervisor
6258 .get(selector)
6259 .map(|module| module.module_id().to_owned()),
6260 ops::SUPERVISOR_HEALTH_PROBE => self
6261 .registry
6262 .get_module(selector)
6263 .ok()
6264 .flatten()
6265 .map(|registration| registration.manifest.module_id),
6266 _ => None,
6267 }
6268 }
6269
6270 async fn handle_supervisor_terminals(
6271 &self,
6272 ctx: &RouteCtx,
6273 frame: Frame,
6274 module_id: String,
6275 ) -> Result<Vec<Frame>, RouterError> {
6276 let Some(module) = self.supervisor.get(&module_id) else {
6277 return Ok(vec![control_error_frame(
6278 &frame,
6279 "unknown_module",
6280 format!("module_id '{module_id}' is not supervised"),
6281 )?]);
6282 };
6283
6284 let terminals = self
6287 .observe_control_wait(
6288 ctx,
6289 Some(module.module_id()),
6290 ops::SUPERVISOR_TERMINALS,
6291 module.read_durable_terminal_history(),
6292 )
6293 .await
6294 .map_err(|error| {
6295 RouterError::backend(
6296 0,
6297 frame.header.corr,
6298 format!("failed to read terminal history: {error}"),
6299 )
6300 })?;
6301 let response = ClientControlResponse::SupervisorTerminals {
6302 module_id,
6303 terminals,
6304 };
6305 Ok(vec![control_response_body_frame(
6306 &frame,
6307 &response,
6308 "ClientControlResponse::SupervisorTerminals",
6309 )?])
6310 }
6311
6312 fn handle_supervisor_routes(
6313 &self,
6314 frame: Frame,
6315 module_id: Option<String>,
6316 ) -> Result<Vec<Frame>, RouterError> {
6317 let modules = self
6318 .forwarding
6319 .route_census(module_id.as_deref())
6320 .map_err(RouterError::Forwarding)?
6321 .into_iter()
6322 .map(|(module_id, routes)| SupervisorRouteModule {
6323 module_id,
6324 routes: routes
6325 .into_iter()
6326 .map(|route| SupervisorRoute {
6327 consumer: match route.principal {
6328 Principal::Reserved { module_id } => {
6329 SupervisorRouteConsumer::Reserved { module_id }
6330 }
6331 Principal::Direct | Principal::Unverified => {
6332 SupervisorRouteConsumer::Direct {
6333 connection_id: route.goodbye_target.connection_id.get(),
6334 }
6335 }
6336 },
6337 age_ms: Instant::now()
6338 .saturating_duration_since(route.bound_at)
6339 .as_millis()
6340 .try_into()
6341 .unwrap_or(u64::MAX),
6342 draining: route.draining,
6343 drain_reason: route.drain_reason,
6344 })
6345 .collect(),
6346 })
6347 .collect();
6348 let response = ClientControlResponse::SupervisorRoutes { modules };
6349 Ok(vec![control_response_body_frame(
6350 &frame,
6351 &response,
6352 "ClientControlResponse::SupervisorRoutes",
6353 )?])
6354 }
6355
6356 async fn handle_supervisor_provenance(
6357 &self,
6358 ctx: &RouteCtx,
6359 frame: Frame,
6360 module_id: Option<String>,
6361 ) -> Result<Vec<Frame>, RouterError> {
6362 let targeted = module_id.is_some();
6363 let selected = if let Some(module_id) = module_id {
6364 let Some(module) = self.supervisor.get(&module_id) else {
6365 return Ok(vec![control_error_frame(
6366 &frame,
6367 "unknown_module",
6368 format!("module_id '{module_id}' is not supervised"),
6369 )?]);
6370 };
6371 vec![module]
6372 } else {
6373 self.supervisor.list()
6374 };
6375
6376 let target = targeted.then(|| selected[0].module_id());
6377 self.observe_control_wait(ctx, target, ops::SUPERVISOR_PROVENANCE, async {
6378 let mut modules = Vec::with_capacity(selected.len());
6379 for module in &selected {
6380 let (status, observed_image) = module
6381 .status_and_running_image_agreement()
6382 .await
6383 .map_err(|err| {
6384 RouterError::backend(
6385 0,
6386 frame.header.corr,
6387 format!("failed to read supervisor status: {err}"),
6388 )
6389 })?;
6390 let module_declared = self
6391 .registry
6392 .get_module(&status.module_id)
6393 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
6394 .and_then(|registration| registration.manifest.provenance)
6395 .map(|build| ModuleDeclaredProvenance::Reported { build })
6396 .unwrap_or(ModuleDeclaredProvenance::Unverifiable);
6397 #[cfg(test)]
6398 let running_image = match &self.provenance_probe_override {
6399 Some(result) => result.clone(),
6400 None => observed_image,
6401 };
6402 #[cfg(not(test))]
6403 let running_image = observed_image;
6404 modules.push(SupervisorModuleProvenance {
6405 module_id: status.module_id,
6406 module_declared,
6407 daemon_observed: SupervisorObservedProcess {
6408 launch_nonce_source: status.launch_nonce_source,
6409 pid: status.pid,
6410 spawned_at_ms: status.spawned_at_ms,
6411 spawned_from: status.spawned_from,
6412 running_image,
6413 },
6414 });
6415 }
6416 let daemon = SupervisorDaemonProvenance {
6417 daemon_build: self.daemon_provenance.build.clone(),
6418 daemon_observed: DaemonObservedProcess {
6419 pid: self.daemon_provenance.pid,
6420 started_at_ms: self
6421 .daemon_provenance
6422 .start_clock
6423 .map(|clock| clock.started_at_ms())
6424 .or(self.daemon_provenance.started_at_ms),
6425 running_image: self
6426 .daemon_provenance
6427 .probe
6428 .observe(
6429 self.daemon_provenance.pid,
6430 self.daemon_provenance.executable_path.as_deref(),
6431 self.daemon_provenance.executable_identity.clone(),
6432 self.daemon_provenance.process_start_time,
6433 )
6434 .await,
6435 },
6436 };
6437 let response = ClientControlResponse::SupervisorProvenance { daemon, modules };
6438 Ok(vec![control_response_body_frame(
6439 &frame,
6440 &response,
6441 "ClientControlResponse::SupervisorProvenance",
6442 )?])
6443 })
6444 .await
6445 }
6446
6447 fn handle_supervisor_health(&self, frame: Frame) -> Result<Vec<Frame>, RouterError> {
6448 self.refresh_capability_requirements();
6449 let generation = self
6450 .registry
6451 .generation()
6452 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?;
6453 let modules = self
6454 .supervisor
6455 .list()
6456 .into_iter()
6457 .map(|module| {
6458 let status = module.status_for_control("health").map_err(|err| {
6459 RouterError::backend(
6460 0,
6461 frame.header.corr,
6462 format!("failed to read supervisor health: {err}"),
6463 )
6464 })?;
6465 let module_id = status.module_id;
6466 let capability_detail = self
6467 .capability_evaluator
6468 .required_problem_detail(&module_id);
6469 Ok(SupervisorHealthEntry {
6470 module_id,
6471 status: status.health.status,
6472 detail: append_capability_problem_detail(
6473 status.health.detail,
6474 capability_detail,
6475 ),
6476 metrics: status.health.metrics,
6477 consecutive_failures: status.health.consecutive_failures,
6478 late_answer_count: status.health.late_answer_count,
6479 last_late_answer_latency_ms: status.health.last_late_answer_latency_ms,
6480 last_action: status.health.last_action,
6481 last_action_ms: status.health.last_action_ms,
6482 last_probe_ms: status.health.last_probe_ms,
6483 })
6484 })
6485 .collect::<Result<Vec<_>, RouterError>>()?;
6486 let response = ClientControlResponse::SupervisorHealth {
6487 generation,
6488 modules,
6489 };
6490 Ok(vec![control_response_body_frame(
6491 &frame,
6492 &response,
6493 "ClientControlResponse::SupervisorHealth",
6494 )?])
6495 }
6496
6497 async fn handle_supervisor_restart(
6498 &self,
6499 frame: Frame,
6500 module_id: String,
6501 drain_timeout_ms: Option<u64>,
6502 requester: Principal,
6503 ) -> Result<Vec<Frame>, RouterError> {
6504 let operation_lock = self.supervisor.operation_lock();
6505 let _operation_guard = operation_lock.lock().await;
6506 let Some(module) = self.supervisor.get(&module_id) else {
6507 return Ok(vec![control_error_frame(
6508 &frame,
6509 "unknown_module",
6510 format!("module_id '{module_id}' is not supervised"),
6511 )?]);
6512 };
6513
6514 self.route_outages.mark_operator_action(&module_id);
6515 if let Err(err) = module
6516 .restart_requested(drain_timeout_ms, Some(requester))
6517 .await
6518 {
6519 self.route_outages
6520 .operator_action_ended_unrefused(&module_id);
6521 let (code, message) = match err {
6522 crate::supervise::SuperviseError::Disabled { .. } => {
6523 ("module_disabled", err.to_string())
6524 }
6525 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6526 ("swap_in_progress", err.to_string())
6527 }
6528 _ => (
6529 "target_unavailable",
6530 format!("failed to restart module_id '{module_id}': {err}"),
6531 ),
6532 };
6533 return Ok(vec![control_error_frame(&frame, code, message)?]);
6534 }
6535
6536 let response = ClientControlResponse::SupervisorAck {
6537 module_id,
6538 applied: true,
6539 };
6540 Ok(vec![control_response_body_frame(
6541 &frame,
6542 &response,
6543 "ClientControlResponse::SupervisorAck",
6544 )?])
6545 }
6546
6547 async fn handle_supervisor_swap(
6551 &self,
6552 frame: Frame,
6553 module_id: String,
6554 ready_timeout_ms: Option<u64>,
6555 requester: Principal,
6556 ) -> Result<Vec<Frame>, RouterError> {
6557 let module = {
6564 let operation_lock = self.supervisor.operation_lock();
6565 let _operation_guard = operation_lock.lock().await;
6566 self.supervisor.get(&module_id)
6567 };
6568 let Some(module) = module else {
6569 return Ok(vec![control_error_frame(
6570 &frame,
6571 "unknown_module",
6572 format!("module_id '{module_id}' is not supervised"),
6573 )?]);
6574 };
6575
6576 self.route_outages.mark_operator_action(&module_id);
6577 if let Err(err) = module
6578 .swap_requested(ready_timeout_ms.map(Duration::from_millis), Some(requester))
6579 .await
6580 {
6581 self.route_outages
6582 .operator_action_ended_unrefused(&module_id);
6583 use crate::supervise::SuperviseError;
6584 let message = err.to_string();
6585 let error = match err {
6586 SuperviseError::Disabled { .. } => ErrorBody::new("module_disabled", message),
6587 SuperviseError::SwapRefused { reason, .. } => ErrorBody {
6588 code: "swap_refused".to_string(),
6589 message,
6590 detail: Some(serde_json::json!({ "reason": reason.as_str() })),
6591 },
6592 SuperviseError::SwapFailed {
6593 arm,
6594 candidate_exit,
6595 ..
6596 } => ErrorBody {
6597 code: "swap_failed".to_string(),
6598 message,
6599 detail: Some(serde_json::json!({
6600 "arm": arm.as_str(),
6601 "candidate_exit_code": candidate_exit.as_ref().and_then(|exit| exit.code),
6602 "candidate_exit_signal": candidate_exit.as_ref().and_then(|exit| exit.signal),
6603 })),
6604 },
6605 _ => ErrorBody::new(
6606 "target_unavailable",
6607 format!("failed to swap module_id '{module_id}': {message}"),
6608 ),
6609 };
6610 return Ok(vec![control_error_body_frame(&frame, error)?]);
6611 }
6612 self.route_outages
6616 .operator_action_ended_unrefused(&module_id);
6617
6618 let response = ClientControlResponse::SupervisorAck {
6619 module_id,
6620 applied: true,
6621 };
6622 Ok(vec![control_response_body_frame(
6623 &frame,
6624 &response,
6625 "ClientControlResponse::SupervisorAck",
6626 )?])
6627 }
6628
6629 async fn handle_supervisor_reload(
6630 &self,
6631 frame: Frame,
6632 module_id: String,
6633 requester: Principal,
6634 ) -> Result<Vec<Frame>, RouterError> {
6635 let operation_lock = self.supervisor.operation_lock();
6636 let _operation_guard = operation_lock.lock().await;
6637 let Some(module) = self.supervisor.get(&module_id) else {
6638 return Ok(vec![control_error_frame(
6639 &frame,
6640 "unknown_module",
6641 format!("module_id '{module_id}' is not supervised"),
6642 )?]);
6643 };
6644
6645 self.route_outages.mark_operator_action(&module_id);
6646 if let Err(err) = module.reload_requested(Some(requester)).await {
6647 self.route_outages
6648 .operator_action_ended_unrefused(&module_id);
6649 let (code, message) = match err {
6650 crate::supervise::SuperviseError::Disabled { .. } => {
6651 ("module_disabled", err.to_string())
6652 }
6653 crate::supervise::SuperviseError::SwapInProgress { .. } => {
6654 ("swap_in_progress", err.to_string())
6655 }
6656 _ => (
6657 "reload_failed",
6658 format!("failed to reload module_id '{module_id}': {err}"),
6659 ),
6660 };
6661 return Ok(vec![control_error_frame(&frame, code, message)?]);
6662 }
6663
6664 let response = ClientControlResponse::SupervisorAck {
6665 module_id,
6666 applied: true,
6667 };
6668 Ok(vec![control_response_body_frame(
6669 &frame,
6670 &response,
6671 "ClientControlResponse::SupervisorAck",
6672 )?])
6673 }
6674
6675 async fn handle_supervisor_rescan(
6679 &self,
6680 frame: Frame,
6681 mode: RescanMode,
6682 requester: Principal,
6683 ) -> Result<Vec<Frame>, RouterError> {
6684 let preview = mode.previews();
6685 let Some(context) = self.rescan.clone() else {
6686 return Ok(vec![control_error_frame(
6687 &frame,
6688 "rescan_unavailable",
6689 "the daemon was not started with a reloadable config path".to_string(),
6690 )?]);
6691 };
6692
6693 let operation_lock = self.supervisor.operation_lock();
6694 let _operation_guard = operation_lock.lock().await;
6695 let loaded = match self.load_rescan_config(&mode, &context) {
6699 Ok(config) => config,
6700 Err((code, message)) => return Ok(vec![control_error_frame(&frame, code, message)?]),
6701 };
6702 let Some((config, activity_config)) = loaded else {
6710 return Ok(vec![control_error_frame(
6711 &frame,
6712 "invalid_daemon_config",
6713 format!(
6714 "daemon config not found at {}; refusing to rescan (an absent config would \
6715 retire every supervised module)",
6716 context.config_path.display()
6717 ),
6718 )?]);
6719 };
6720 let (
6721 configured_port,
6722 storage_config,
6723 admission_facts_carrier_module_id,
6724 admission_facts_targets,
6725 scope_authority_owners,
6726 federated_scope_owner,
6727 privileged_operator_prompt,
6728 modules,
6729 reserved_capabilities,
6730 ) = (
6731 config.port,
6732 config.storage,
6733 config.admission_facts_carrier_module_id,
6734 config.admission_facts_targets,
6735 config.scope_authority_owners,
6736 config.federated_scope_owner,
6737 config.privileged_operator_prompt,
6738 config.modules,
6739 config.reserved_capabilities,
6740 );
6741
6742 let mut restart_required = Vec::new();
6751 for section in RestartRequiredSection::ALL {
6752 let changed = match section {
6753 RestartRequiredSection::Port => configured_port != context.configured_port,
6754 RestartRequiredSection::Storage => storage_config != context.storage_config,
6755 RestartRequiredSection::AdmissionFactsCarrierModuleId => {
6756 admission_facts_carrier_module_id != context.admission_facts_carrier_module_id
6757 }
6758 RestartRequiredSection::AdmissionFactsTargets => {
6759 admission_facts_targets != context.admission_facts_targets
6760 }
6761 RestartRequiredSection::ScopeAuthorityOwners => {
6762 scope_authority_owners != context.scope_authority_owners
6763 }
6764 RestartRequiredSection::FederatedScopeOwner => {
6765 federated_scope_owner != context.federated_scope_owner
6766 }
6767 RestartRequiredSection::PrivilegedOperatorPrompt => {
6768 privileged_operator_prompt != context.privileged_operator_prompt
6769 }
6770 };
6771 if changed {
6772 restart_required.push(section.label().to_string());
6773 }
6774 }
6775 if !restart_required.is_empty() {
6776 warn!(
6777 config_path = %context.config_path.display(),
6778 sections = %restart_required.join(", "),
6779 "daemon config changed outside the modules section; restart the daemon to apply those changes"
6780 );
6781 }
6782
6783 for configured in &modules {
6784 if let Err(err) = validate_spec(&configured.module_spec()) {
6785 return Ok(vec![control_error_frame(
6786 &frame,
6787 "invalid_daemon_config",
6788 format!("supervisor rescan rejected daemon config: {err}"),
6789 )?]);
6790 }
6791 }
6792
6793 let configured_capabilities = modules
6794 .iter()
6795 .map(|module| (module.module_id.clone(), module.enabled))
6796 .collect::<Vec<_>>();
6797 let preview_capability_warnings = if preview {
6798 let (_, registrations) = self.runtime_capability_snapshot()?;
6799 let current_modules = self
6800 .supervisor
6801 .list()
6802 .into_iter()
6803 .map(|module| module.module_id().to_string())
6804 .collect::<BTreeSet<_>>();
6805 let resulting_modules = configured_capabilities.clone();
6806 let removed = current_modules
6807 .into_iter()
6808 .filter(|module_id| {
6809 !resulting_modules
6810 .iter()
6811 .any(|(configured_id, _)| configured_id == module_id)
6812 })
6813 .collect::<Vec<_>>();
6814 let disabled = match &mode {
6817 RescanMode::Candidate(_) => resulting_modules
6818 .iter()
6819 .filter(|(_, enabled)| !enabled)
6820 .map(|(module_id, _)| module_id.clone())
6821 .collect::<Vec<_>>(),
6822 _ => Vec::new(),
6823 };
6824 self.capability_evaluator.preview_warnings(
6825 resulting_modules,
6826 &removed,
6827 &disabled,
6828 ®istrations,
6829 )
6830 } else {
6831 Vec::new()
6832 };
6833 let reconciled = match &mode {
6834 RescanMode::Candidate(candidate) => self
6835 .preview_candidate(&context, modules, candidate)
6836 .map(|result| (result, Vec::new())),
6837 _ => {
6838 self.reconcile_supervised_modules(
6839 &context.supervisor,
6840 modules,
6841 preview,
6842 requester.clone(),
6843 )
6844 .await
6845 }
6846 };
6847 let (result, changed) = match reconciled {
6848 Ok(result) => result,
6849 Err(message) => {
6850 return Ok(vec![control_error_frame(&frame, "rescan_failed", message)?])
6851 }
6852 };
6853 if !preview {
6854 if let Some(ledger) = &self.activity {
6855 ledger.module_rescanned(&result.added, &result.removed, requester.clone());
6856 ledger.config_changed(&result.added, &result.removed, &changed, requester);
6857 }
6858 if let Err(code) = self.configure_activity(activity_config).await {
6859 warn!(
6860 code,
6861 "activity retention could not be applied during rescan"
6862 );
6863 }
6864 self.capability_evaluator
6865 .configure(configured_capabilities, reserved_capabilities);
6866 self.capability_evaluator.wake_deadline_loop();
6867 self.refresh_capability_requirements();
6868 }
6869 let mut result = result;
6870 result.restart_required = restart_required;
6871 result.capability_warnings = preview_capability_warnings;
6872 let response = ClientControlResponse::SupervisorRescan { result };
6873 Ok(vec![control_response_body_frame(
6874 &frame,
6875 &response,
6876 "ClientControlResponse::SupervisorRescan",
6877 )?])
6878 }
6879
6880 async fn handle_supervisor_release_reserved(
6881 &self,
6882 frame: Frame,
6883 module_id: String,
6884 ) -> Result<Vec<Frame>, RouterError> {
6885 let Some(context) = self.rescan.clone() else {
6886 return Ok(vec![control_error_frame(
6887 &frame,
6888 "release_unavailable",
6889 "reserved-id release requires a daemon started with a reloadable config path",
6890 )?]);
6891 };
6892 let operation_lock = self.supervisor.operation_lock();
6893 let _operation_guard = operation_lock.lock().await;
6894 let loaded = match crate::daemon_config::load(&context.config_path) {
6895 Ok(Some(config)) => config,
6896 Ok(None) => {
6897 return Ok(vec![control_error_frame(
6898 &frame,
6899 "invalid_daemon_config",
6900 format!(
6901 "daemon config not found at {}; refusing to release reserved module_id '{module_id}'",
6902 context.config_path.display()
6903 ),
6904 )?])
6905 }
6906 Err(err) => {
6907 return Ok(vec![control_error_frame(
6908 &frame,
6909 "invalid_daemon_config",
6910 format!("unable to verify reserved-id release against daemon config: {err}"),
6911 )?])
6912 }
6913 };
6914 if loaded
6915 .modules
6916 .iter()
6917 .any(|configured| configured.module_id == module_id)
6918 {
6919 return Ok(vec![control_error_frame(
6920 &frame,
6921 "reserved_module_configured",
6922 format!(
6923 "module_id '{module_id}' remains configured; remove its config entry and rescan before releasing its reserved id"
6924 ),
6925 )?]);
6926 }
6927 if !self.supervisor.release_retained_reserved_gate(&module_id) {
6928 return Ok(vec![control_error_frame(
6929 &frame,
6930 "reserved_gate_not_retained",
6931 format!(
6932 "module_id '{module_id}' has no retired reserved-id gate to release; rescan its removed reserved configuration first"
6933 ),
6934 )?]);
6935 }
6936
6937 let response = ClientControlResponse::SupervisorAck {
6938 module_id,
6939 applied: true,
6940 };
6941 Ok(vec![control_response_body_frame(
6942 &frame,
6943 &response,
6944 "ClientControlResponse::SupervisorAck",
6945 )?])
6946 }
6947
6948 fn plan_rescan(
6952 &self,
6953 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
6954 ) -> Result<RescanPlan, String> {
6955 let mut current = BTreeMap::new();
6956 for module in self.supervisor.list() {
6957 let (spec, health) = module.configuration().map_err(|err| {
6958 format!(
6959 "failed to read configuration for module_id '{}': {err}",
6960 module.module_id()
6961 )
6962 })?;
6963 let enabled = module
6964 .status()
6965 .map_err(|err| {
6966 format!(
6967 "failed to read status for module_id '{}': {err}",
6968 module.module_id()
6969 )
6970 })?
6971 .enabled;
6972 current.insert(
6973 module.module_id().to_string(),
6974 (module, spec, health, enabled),
6975 );
6976 }
6977 let configured = configured_modules
6978 .into_iter()
6979 .map(|module| (module.module_id.clone(), module))
6980 .collect::<BTreeMap<_, _>>();
6981
6982 let added = configured
6983 .keys()
6984 .filter(|module_id| !current.contains_key(*module_id))
6985 .cloned()
6986 .collect::<Vec<_>>();
6987 let removed = current
6988 .keys()
6989 .filter(|module_id| !configured.contains_key(*module_id))
6990 .cloned()
6991 .collect::<Vec<_>>();
6992 let mut changed_pending_reload = Vec::new();
6993 let mut configuration_changes = BTreeSet::new();
6994 let mut enabled_changes = BTreeSet::new();
6995 let mut unchanged = 0_u32;
6996
6997 for (module_id, configured_module) in &configured {
6998 let Some((_, current_spec, current_health, current_enabled)) = current.get(module_id)
6999 else {
7000 continue;
7001 };
7002 let launch_changed = *current_spec != configured_module.module_spec();
7006 let configuration_changed =
7007 launch_changed || *current_health != configured_module.health;
7008 let enabled_changed = *current_enabled != configured_module.enabled
7011 || self
7012 .capability_evaluator
7013 .configured_enabled(module_id)
7014 .is_some_and(|enabled| enabled != configured_module.enabled);
7015 if configuration_changed {
7016 configuration_changes.insert(module_id.clone());
7017 }
7018 if launch_changed {
7019 changed_pending_reload.push(module_id.clone());
7020 }
7021 if enabled_changed {
7022 enabled_changes.insert(module_id.clone());
7023 }
7024 if !configuration_changed && !enabled_changed {
7025 unchanged = unchanged.saturating_add(1);
7026 }
7027 }
7028 Ok(RescanPlan {
7029 current,
7030 configured,
7031 added,
7032 removed,
7033 changed_pending_reload,
7034 configuration_changes,
7035 enabled_changes,
7036 unchanged,
7037 })
7038 }
7039
7040 async fn reconcile_supervised_modules(
7049 &self,
7050 supervisor: &Supervisor,
7051 configured_modules: Vec<crate::daemon_config::ConfiguredModule>,
7052 preview: bool,
7053 requester: Principal,
7054 ) -> Result<(SupervisorRescanResult, Vec<String>), String> {
7055 let plan = self.plan_rescan(configured_modules)?;
7056
7057 if preview {
7063 return Ok((plan.preview_result(), Vec::new()));
7064 }
7065 let changed = plan
7066 .configuration_changes
7067 .union(&plan.enabled_changes)
7068 .cloned()
7069 .collect::<Vec<_>>();
7070 let RescanPlan {
7071 current,
7072 configured,
7073 added,
7074 removed,
7075 changed_pending_reload,
7076 configuration_changes,
7077 enabled_changes,
7078 unchanged,
7079 } = plan;
7080 if !(added.is_empty()
7085 && removed.is_empty()
7086 && configuration_changes.is_empty()
7087 && enabled_changes.is_empty())
7088 {
7089 if let Some(context) = &self.rescan {
7090 context
7091 .generation
7092 .fetch_add(1, std::sync::atomic::Ordering::SeqCst);
7093 }
7094 }
7095
7096 for module_id in &removed {
7097 let module = ¤t
7098 .get(module_id)
7099 .expect("removed module came from current supervisor state")
7100 .0;
7101 module
7102 .retire_requested(Some(requester.clone()))
7103 .await
7104 .map_err(|err| {
7105 format!("failed to retire module_id '{module_id}' during rescan: {err}")
7106 })?;
7107 self.supervisor.record_rescan_removal(module_id);
7141 self.supervisor.retire(module_id);
7142 self.route_outages.forget(module_id);
7143 }
7144
7145 for module_id in configured.keys() {
7146 let Some((module, _, _, current_enabled)) = current.get(module_id) else {
7147 continue;
7148 };
7149 let configured_module = configured
7150 .get(module_id)
7151 .expect("configured module id came from configured map");
7152 if configuration_changes.contains(module_id) {
7153 module
7154 .update_configuration(
7155 configured_module.module_spec(),
7156 configured_module.health.clone(),
7157 configured_module.drain_timeout_ms,
7158 )
7159 .await
7160 .map_err(|err| {
7161 format!(
7162 "failed to update module_id '{module_id}' configuration during rescan: {err}"
7163 )
7164 })?;
7165 }
7166 if *current_enabled != configured_module.enabled {
7169 self.route_outages.mark_operator_action(module_id);
7172 module
7173 .set_enabled_requested(configured_module.enabled, Some(requester.clone()))
7174 .await
7175 .map_err(|err| {
7176 self.route_outages.operator_action_ended_unrefused(module_id);
7177 format!(
7178 "failed to apply module_id '{module_id}' enabled={} during rescan: {err}",
7179 configured_module.enabled
7180 )
7181 })?;
7182 }
7183 }
7184
7185 for module_id in &added {
7186 let configured_module = configured
7187 .get(module_id)
7188 .expect("added module id came from configured map");
7189 supervisor
7190 .supervise_configured_requested(
7191 configured_module.module_spec(),
7192 configured_module.enabled,
7193 configured_module.health.clone(),
7194 configured_module.drain_timeout_ms,
7195 configured_module.restart,
7196 configured_module.register_timeout,
7197 Some(requester.clone()),
7198 )
7199 .map_err(|err| {
7200 format!("failed to add module_id '{module_id}' during rescan: {err}")
7201 })?;
7202 }
7203
7204 Ok((
7205 SupervisorRescanResult {
7206 added,
7207 removed,
7208 changed_pending_reload,
7209 enabled_changes: enabled_changes.iter().cloned().collect(),
7210 unchanged,
7211 preview: false,
7212 restart_required: Vec::new(),
7215 capability_warnings: Vec::new(),
7216 module_actions: Vec::new(),
7217 earlier_pending: Vec::new(),
7218 expect: None,
7219 },
7220 changed,
7221 ))
7222 }
7223
7224 async fn handle_supervisor_set_enabled(
7225 &self,
7226 frame: Frame,
7227 module_id: String,
7228 enabled: bool,
7229 requester: Principal,
7230 ) -> Result<Vec<Frame>, RouterError> {
7231 let operation_lock = self.supervisor.operation_lock();
7232 let _operation_guard = operation_lock.lock().await;
7233 let Some(module) = self.supervisor.get(&module_id) else {
7234 return Ok(vec![control_error_frame(
7235 &frame,
7236 "unknown_module",
7237 format!("module_id '{module_id}' is not supervised"),
7238 )?]);
7239 };
7240
7241 self.route_outages.mark_operator_action(&module_id);
7244 let applied = match module.set_enabled_requested(enabled, Some(requester)).await {
7245 Ok(applied) => applied,
7246 Err(err) => {
7247 self.route_outages
7248 .operator_action_ended_unrefused(&module_id);
7249 return Ok(vec![control_error_frame(
7250 &frame,
7251 "target_unavailable",
7252 format!("failed to set module_id '{module_id}' enabled={enabled}: {err}"),
7253 )?]);
7254 }
7255 };
7256 if !applied {
7257 self.route_outages
7260 .operator_action_ended_unrefused(&module_id);
7261 }
7262
7263 self.capability_evaluator.wake_deadline_loop();
7264 self.refresh_capability_requirements();
7265 let response = ClientControlResponse::SupervisorAck { module_id, applied };
7266 Ok(vec![control_response_body_frame(
7267 &frame,
7268 &response,
7269 "ClientControlResponse::SupervisorAck",
7270 )?])
7271 }
7272
7273 async fn handle_supervisor_health_probe(
7274 &self,
7275 ctx: &RouteCtx,
7276 frame: Frame,
7277 module_id: String,
7278 ) -> Result<Vec<Frame>, RouterError> {
7279 self.refresh_capability_requirements();
7280 let Some(registration) = self
7281 .registry
7282 .get_module(&module_id)
7283 .map_err(|err| RouterError::backend(0, frame.header.corr, err.to_string()))?
7284 else {
7285 return Ok(vec![control_error_frame(
7286 &frame,
7287 "unknown_module",
7288 format!("module_id '{module_id}' is not registered"),
7289 )?]);
7290 };
7291
7292 if !module_registration_grants_op(®istration.control_ops, MODULE_CONTROL_OP_HEALTH_CHECK)
7305 {
7306 return Ok(vec![control_error_frame(
7307 &frame,
7308 "health_not_advertised",
7309 format!("module_id '{module_id}' did not advertise health.check"),
7310 )?]);
7311 }
7312
7313 self.observe_control_wait(
7314 ctx,
7315 Some(®istration.manifest.module_id),
7316 ops::SUPERVISOR_HEALTH_PROBE,
7317 async {
7318 let deadline = Instant::now() + self.health_probe_timeout;
7319 let pending = match self.forwarding.begin_module_control_rpc_for(
7320 &module_id,
7321 MODULE_CONTROL_OP_HEALTH_CHECK,
7322 deadline,
7323 ) {
7324 Ok(pending) => pending,
7325 Err(err) => {
7326 return Ok(vec![control_error_frame(
7327 &frame,
7328 forwarding_error_code(&err),
7329 err.to_string(),
7330 )?])
7331 }
7332 };
7333
7334 let PendingModuleControlRpc {
7335 endpoint,
7336 module_sink,
7337 negotiated_ver,
7338 corr: probe_corr,
7339 receiver,
7340 } = pending;
7341 let mut guard =
7342 ModuleControlRpcGuard::new(Arc::clone(&self.forwarding), endpoint, probe_corr);
7343 let probe_body = serde_json::to_vec(&ModuleControlRequest::HealthCheck {})
7344 .map_err(|err| {
7345 RouterError::backend(
7346 0,
7347 frame.header.corr,
7348 format!("failed to encode health.check request: {err}"),
7349 )
7350 })?;
7351 let probe_frame = Frame::build_with_version(
7352 negotiated_ver,
7353 FrameType::Request,
7354 control_flags(),
7355 0,
7356 0,
7357 probe_corr,
7358 probe_body,
7359 )
7360 .map_err(RouterError::FrameBuild)?;
7361
7362 match timeout_at(deadline, module_sink.send(probe_frame)).await {
7365 Ok(Ok(())) => {}
7366 Ok(Err(err)) => {
7367 return Ok(vec![control_error_frame(
7368 &frame,
7369 "target_unavailable",
7370 err.to_string(),
7371 )?]);
7372 }
7373 Err(_) => {
7374 return Ok(vec![control_error_frame(
7375 &frame,
7376 "module_timeout",
7377 format!(
7378 "module_id '{module_id}' did not answer health.check within {:?}",
7379 self.health_probe_timeout
7380 ),
7381 )?]);
7382 }
7383 }
7384
7385 match timeout_at(deadline, receiver).await {
7386 Ok(Ok(ModuleControlRpcOutcome::Response(response))) => {
7387 guard.disarm();
7388 let Some(report) = response.health_report() else {
7389 return Ok(vec![control_error_frame(
7390 &frame,
7391 "invalid_control_body",
7392 "health.check RPC returned a non-health response",
7393 )?]);
7394 };
7395 let HealthReport {
7400 status,
7401 detail,
7402 metrics,
7403 } = report;
7404 let capability_detail = self
7405 .capability_evaluator
7406 .required_problem_detail(&module_id);
7407 let response = ClientControlResponse::SupervisorHealthProbe {
7408 module_id,
7409 status,
7410 detail: append_capability_problem_detail(detail, capability_detail),
7411 metrics,
7412 };
7413 Ok(vec![control_response_body_frame(
7414 &frame,
7415 &response,
7416 "ClientControlResponse::SupervisorHealthProbe",
7417 )?])
7418 }
7419 Ok(Ok(ModuleControlRpcOutcome::Rejected(body))) => {
7420 guard.disarm();
7421 Ok(vec![control_relayed_error_body_frame(&frame, body)?])
7426 }
7427 Ok(Ok(ModuleControlRpcOutcome::ModuleGone(message))) => {
7428 guard.disarm();
7429 Ok(vec![control_error_frame(
7430 &frame,
7431 "target_unavailable",
7432 message,
7433 )?])
7434 }
7435 Ok(Ok(ModuleControlRpcOutcome::MalformedResponse(message))) => {
7436 guard.disarm();
7437 Ok(vec![control_error_frame(
7438 &frame,
7439 "invalid_control_body",
7440 message,
7441 )?])
7442 }
7443 Ok(Ok(ModuleControlRpcOutcome::UnexpectedOp { expected, actual })) => {
7444 guard.disarm();
7445 Ok(vec![control_error_frame(
7446 &frame,
7447 "invalid_control_body",
7448 format!("expected module-control op '{expected}', got '{actual}'"),
7449 )?])
7450 }
7451 Ok(Ok(ModuleControlRpcOutcome::DeadlineElapsed)) => {
7452 guard.disarm();
7453 Ok(vec![control_error_frame(
7454 &frame,
7455 "module_timeout",
7456 format!(
7457 "module_id '{module_id}' answered health.check after {:?}",
7458 self.health_probe_timeout
7459 ),
7460 )?])
7461 }
7462 Ok(Err(_)) => Ok(vec![control_error_frame(
7463 &frame,
7464 "target_unavailable",
7465 "health.check waiter was canceled before the module responded",
7466 )?]),
7467 Err(_) => Ok(vec![control_error_frame(
7468 &frame,
7469 "module_timeout",
7470 format!(
7471 "module_id '{module_id}' did not answer health.check within {:?}",
7472 self.health_probe_timeout
7473 ),
7474 )?]),
7475 }
7476 },
7477 )
7478 .await
7479 }
7480
7481 fn supervisor_status(
7482 &self,
7483 module_id: &str,
7484 corr: u64,
7485 ) -> Result<Option<(crate::supervise::ModuleStatus, bool)>, RouterError> {
7486 self.supervisor
7487 .get(module_id)
7488 .map(|module| {
7489 let warming = module.is_warming_for_control("status").map_err(|err| {
7490 RouterError::backend(
7491 0,
7492 corr,
7493 format!(
7494 "failed to read supervisor warming state for module_id '{module_id}': {err}"
7495 ),
7496 )
7497 })?;
7498 module.status_for_control("status").map_err(|err| {
7499 RouterError::backend(
7500 0,
7501 corr,
7502 format!(
7503 "failed to read supervisor status for module_id '{module_id}': {err}"
7504 ),
7505 )
7506 }).map(|status| (status, warming))
7507 })
7508 .transpose()
7509 }
7510
7511 fn guard_module_control_op(
7512 &self,
7513 frame: &Frame,
7514 module_id: &str,
7515 op: &str,
7516 ) -> Result<Option<Frame>, RouterError> {
7517 if self.module_grants_op(module_id, op, frame.header.corr)? {
7518 return Ok(None);
7519 }
7520
7521 Ok(Some(control_error_frame(
7522 frame,
7523 "op_not_allowed",
7524 format!("module_id '{module_id}' did not grant control op '{op}'"),
7525 )?))
7526 }
7527
7528 fn module_grants_op(&self, module_id: &str, op: &str, corr: u64) -> Result<bool, RouterError> {
7529 let Some(registration) = self
7530 .registry
7531 .get_module(module_id)
7532 .map_err(|err| RouterError::backend(0, corr, err.to_string()))?
7533 else {
7534 return Ok(false);
7535 };
7536 Ok(module_registration_grants_op(®istration.control_ops, op))
7537 }
7538
7539 fn handle_status_update(
7540 &self,
7541 endpoint: ModuleEndpointId,
7542 frame: Frame,
7543 ) -> Result<Vec<Frame>, RouterError> {
7544 let update = match serde_json::from_slice::<ModuleControlPush>(&frame.body) {
7545 Ok(update) => update,
7546 Err(err) => {
7547 if is_known_module_push_op(&frame.body) {
7552 return Ok(vec![control_error_frame(
7553 &frame,
7554 "invalid_control_body",
7555 format!("malformed module control push body: {err}"),
7556 )?]);
7557 }
7558 return Ok(Vec::new());
7559 }
7560 };
7561
7562 match update {
7563 ModuleControlPush::RouteStatus {
7564 route_channel,
7565 route_epoch,
7566 status,
7567 } => {
7568 self.forwarding
7569 .cache_status(endpoint, route_channel, route_epoch, status)
7570 .map_err(RouterError::Forwarding)?;
7571 }
7572 }
7573 Ok(Vec::new())
7574 }
7575
7576 fn handle_route_poll(
7577 &self,
7578 ctx: &RouteCtx,
7579 frame: Frame,
7580 route_channel: u16,
7581 route_epoch: u32,
7582 kind: PollKind,
7583 ) -> Result<Vec<Frame>, RouterError> {
7584 let snapshot = self
7585 .forwarding
7586 .route_poll_snapshot(ctx.connection_id, route_channel, route_epoch)
7587 .map_err(RouterError::Forwarding)?;
7588 let response = match (kind, snapshot) {
7589 (PollKind::Status, RoutePollSnapshot::Bound { status, .. }) => {
7590 ClientControlResponse::RoutePoll {
7591 route_channel,
7592 route_epoch,
7593 status,
7594 live: None,
7595 }
7596 }
7597 (PollKind::Status, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7598 route_channel,
7599 route_epoch,
7600 status: None,
7601 live: None,
7602 },
7603 (PollKind::Liveness, RoutePollSnapshot::Bound { module_id, .. }) => {
7604 let live = self
7630 .process_liveness
7631 .as_ref()
7632 .and_then(|source| source.process_live(&module_id))
7633 .unwrap_or(true);
7634 ClientControlResponse::RoutePoll {
7635 route_channel,
7636 route_epoch,
7637 status: None,
7638 live: Some(live),
7639 }
7640 }
7641 (PollKind::Liveness, RoutePollSnapshot::Absent) => ClientControlResponse::RoutePoll {
7642 route_channel,
7643 route_epoch,
7644 status: None,
7645 live: Some(false),
7646 },
7647 };
7648
7649 Ok(vec![control_response_body_frame(
7650 &frame,
7651 &response,
7652 "ClientControlResponse::RoutePoll",
7653 )?])
7654 }
7655
7656 pub(crate) fn observe_module_control_completion(
7657 &self,
7658 completion: ModuleControlRpcCompletion,
7659 ) -> bool {
7660 match completion {
7661 ModuleControlRpcCompletion::Unknown => false,
7662 ModuleControlRpcCompletion::Settled => true,
7663 ModuleControlRpcCompletion::LateHealthAnswer { module_id, latency } => {
7664 let latency_ms = latency.as_millis().min(u128::from(u64::MAX)) as u64;
7665 info!(
7666 module_id = %module_id,
7667 latency_ms,
7668 "late health.check answer proves the module is alive"
7669 );
7670 match self
7671 .supervisor
7672 .record_late_health_answer(&module_id, latency_ms)
7673 {
7674 Ok(true) => {}
7675 Ok(false) => debug!(
7676 module_id = %module_id,
7677 latency_ms,
7678 "late health.check answer has no active supervisor snapshot"
7679 ),
7680 Err(err) => warn!(
7681 module_id = %module_id,
7682 latency_ms,
7683 error = %err,
7684 "failed to record late health.check answer"
7685 ),
7686 }
7687 true
7688 }
7689 }
7690 }
7691
7692 fn refuse_to_end_module_connection_for_a_client(
7713 &self,
7714 module_connection_id: ConnectionId,
7715 corr: u64,
7716 err: ForwardingError,
7717 ) -> Result<(), RouterError> {
7718 if let ForwardingError::ConnectionClosing { connection_id } = err {
7719 if connection_id != module_connection_id {
7720 warn!(
7721 module_connection_id = module_connection_id.get(),
7722 client_connection_id = connection_id.get(),
7723 corr,
7724 "dropping a route.bind response for a closing client; the module connection keeps serving"
7725 );
7726 return Ok(());
7727 }
7728 }
7729 Err(RouterError::Forwarding(err))
7730 }
7731
7732 fn handle_module_relay_response(
7733 &self,
7734 connection_id: ConnectionId,
7735 frame: Frame,
7736 ) -> Result<Vec<Frame>, RouterError> {
7737 let mut secondary_error = None;
7738 let outcome = match frame.header.ty {
7739 FrameType::Response => match serde_json::from_slice::<ControlOpProbe>(&frame.body) {
7740 Ok(probe) if probe.op == "route.bind" => {
7741 match serde_json::from_slice::<ModuleControlResponse>(&frame.body) {
7742 Ok(ModuleControlResponse::RouteBindAck {}) => {
7743 RouteBindRelayOutcome::Accepted
7744 }
7745 Ok(other) => {
7746 let message =
7747 format!("route.bind response carried unexpected body: {other:?}");
7748 secondary_error = Some(control_error_frame(
7749 &frame,
7750 "invalid_control_body",
7751 message.clone(),
7752 )?);
7753 RouteBindRelayOutcome::ModuleGone(message)
7754 }
7755 Err(err) => {
7756 let message = format!("malformed route.bind response body: {err}");
7757 secondary_error = Some(control_error_frame(
7758 &frame,
7759 "invalid_control_body",
7760 message.clone(),
7761 )?);
7762 RouteBindRelayOutcome::ModuleGone(message)
7763 }
7764 }
7765 }
7766 Ok(probe) => {
7767 let outcome = match serde_json::from_slice::<ModuleControlResponse>(&frame.body)
7768 {
7769 Ok(response) => ModuleControlRpcOutcome::Response(response),
7770 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7771 "malformed {} response body: {err}",
7772 probe.op
7773 )),
7774 };
7775 let completion = self
7776 .forwarding
7777 .complete_module_control_rpc(
7778 connection_id,
7779 frame.header.corr,
7780 Some(&probe.op),
7781 outcome,
7782 )
7783 .map_err(RouterError::Forwarding)?;
7784 if !self.observe_module_control_completion(completion) {
7785 debug!(
7786 connection_id = connection_id.get(),
7787 corr = frame.header.corr,
7788 op = %probe.op,
7789 "dropping late or unknown module-control RPC response"
7790 );
7791 }
7792 return Ok(Vec::new());
7793 }
7794 Err(err) => {
7795 if let Some(expected_op) = self
7796 .forwarding
7797 .pending_module_control_op(connection_id, frame.header.corr)
7798 .map_err(RouterError::Forwarding)?
7799 {
7800 let completion = self
7801 .forwarding
7802 .complete_module_control_rpc(
7803 connection_id,
7804 frame.header.corr,
7805 None,
7806 ModuleControlRpcOutcome::MalformedResponse(format!(
7807 "malformed {expected_op} response body: {err}"
7808 )),
7809 )
7810 .map_err(RouterError::Forwarding)?;
7811 if !self.observe_module_control_completion(completion) {
7812 debug!(
7813 connection_id = connection_id.get(),
7814 corr = frame.header.corr,
7815 "dropping late malformed module-control RPC response"
7816 );
7817 }
7818 return Ok(Vec::new());
7819 }
7820 let message = format!("malformed route.bind response body: {err}");
7821 secondary_error = Some(control_error_frame(
7822 &frame,
7823 "invalid_control_body",
7824 message.clone(),
7825 )?);
7826 RouteBindRelayOutcome::ModuleGone(message)
7827 }
7828 },
7829 FrameType::Error => {
7830 if self
7831 .forwarding
7832 .pending_module_control_op(connection_id, frame.header.corr)
7833 .map_err(RouterError::Forwarding)?
7834 .is_some()
7835 {
7836 let outcome = match serde_json::from_slice::<ErrorBody>(&frame.body) {
7837 Ok(body) => ModuleControlRpcOutcome::Rejected(body),
7838 Err(err) => ModuleControlRpcOutcome::MalformedResponse(format!(
7839 "malformed module-control ERROR body: {err}"
7840 )),
7841 };
7842 let completion = self
7843 .forwarding
7844 .complete_module_control_rpc(
7845 connection_id,
7846 frame.header.corr,
7847 None,
7848 outcome,
7849 )
7850 .map_err(RouterError::Forwarding)?;
7851 if !self.observe_module_control_completion(completion) {
7852 debug!(
7853 connection_id = connection_id.get(),
7854 corr = frame.header.corr,
7855 "dropping late or unknown module-control RPC error"
7856 );
7857 }
7858 return Ok(Vec::new());
7859 }
7860 match serde_json::from_slice::<ErrorBody>(&frame.body) {
7861 Ok(body) => RouteBindRelayOutcome::Rejected(body),
7862 Err(err) => {
7863 let message = format!("malformed route.bind ERROR body: {err}");
7864 secondary_error = Some(control_error_frame(
7865 &frame,
7866 "invalid_control_body",
7867 message.clone(),
7868 )?);
7869 RouteBindRelayOutcome::ModuleGone(message)
7870 }
7871 }
7872 }
7873 ty => {
7874 return Ok(vec![control_error_frame(
7875 &frame,
7876 "unsupported_control_frame",
7877 format!("unsupported module channel-0 frame {ty:?}"),
7878 )?])
7879 }
7880 };
7881
7882 let settled =
7883 self.forwarding
7884 .complete_pending_relay(connection_id, frame.header.corr, outcome);
7885 let completion = match settled {
7886 Ok(completion) => completion,
7887 Err(err) => {
7888 self.refuse_to_end_module_connection_for_a_client(
7889 connection_id,
7890 frame.header.corr,
7891 err,
7892 )?;
7893 return Ok(secondary_error.into_iter().collect());
7894 }
7895 };
7896 if let Some(target) = completion.abandoned.as_ref() {
7897 send_goodbye_target_best_effort(&self.counters, target, "late accepted route.bind");
7898 }
7899 if !completion.settled {
7900 debug!(
7901 connection_id = connection_id.get(),
7902 corr = frame.header.corr,
7903 frame_type = ?frame.header.ty,
7904 "dropping late or unknown route.bind relay response"
7905 );
7906 }
7907 Ok(secondary_error.into_iter().collect())
7908 }
7909
7910 fn handle_goodbye(&self, connection_id: ConnectionId) -> Result<Vec<Frame>, RouterError> {
7911 debug!(connection_id = connection_id.get(), "handling GOODBYE");
7912 self.cleanup_connection_with_end_reason(
7916 connection_id,
7917 RegistrationEndReason::ExplicitGoodbye,
7918 )
7919 .map_err(|err| RouterError::backend(0, 0, err.to_string()))?;
7920 Ok(Vec::new())
7921 }
7922}
7923
7924impl Default for ControlHandler {
7925 fn default() -> Self {
7926 Self::new(Arc::new(Registry::default()))
7927 }
7928}
7929
7930impl crate::supervise::SwapPromotionObserver for ControlHandler {
7931 fn swap_promoted(&self, registration: &crate::registry::ModuleRegistration) {
7932 self.apply_registration_capabilities(registration);
7933 }
7934}
7935
7936fn capability_requirement_status(status: RequirementStatus) -> CapabilityRequirementStatus {
7937 CapabilityRequirementStatus {
7938 consumer: status.consumer,
7939 capability: status.capability,
7940 need: match status.need {
7941 subc_protocol::manifest::CapabilityNeed::Required => "required".to_string(),
7942 subc_protocol::manifest::CapabilityNeed::Optional => "optional".to_string(),
7943 },
7944 verdict: status.verdict.as_str().to_string(),
7945 episode_seq: status.episode_seq,
7946 config_satisfiable: status.config_satisfiable,
7947 runtime_available: status.runtime_available,
7948 detail: status.detail,
7949 }
7950}
7951
7952fn append_capability_problem_detail(
7953 detail: Option<String>,
7954 capability_detail: Option<String>,
7955) -> Option<String> {
7956 match (detail, capability_detail) {
7957 (Some(detail), Some(capability_detail)) => Some(format!("{detail}; {capability_detail}")),
7958 (Some(detail), None) => Some(detail),
7959 (None, Some(capability_detail)) => Some(capability_detail),
7960 (None, None) => None,
7961 }
7962}
7963
7964fn subc_ops() -> Vec<String> {
7965 SUBC_CONTROL_OPS
7966 .iter()
7967 .map(|op| (*op).to_string())
7968 .collect()
7969}
7970
7971fn module_subc_ops() -> Vec<String> {
7972 SUBC_CONTROL_OPS
7973 .iter()
7974 .chain(MODULE_TO_SUBC_CONTROL_OPS.iter())
7975 .map(|op| (*op).to_string())
7976 .collect()
7977}
7978
7979#[cfg(test)]
7980fn module_baseline_control_ops() -> Vec<String> {
7981 MODULE_BASELINE_CONTROL_OPS
7982 .iter()
7983 .map(|op| (*op).to_string())
7984 .collect()
7985}
7986
7987fn effective_module_control_ops(declared: Option<Vec<String>>) -> Vec<String> {
7988 let mut seen = HashSet::new();
7989 let mut effective = Vec::new();
7990 for op in MODULE_BASELINE_CONTROL_OPS {
7991 if seen.insert((*op).to_string()) {
7992 effective.push((*op).to_string());
7993 }
7994 }
7995 for op in declared.unwrap_or_default() {
7996 if seen.insert(op.clone()) {
7997 effective.push(op);
7998 }
7999 }
8000 effective
8001}
8002
8003fn module_registration_grants_op(control_ops: &[String], op: &str) -> bool {
8004 MODULE_BASELINE_CONTROL_OPS.contains(&op) || control_ops.iter().any(|granted| granted == op)
8005}
8006
8007fn target_module_id(target: &RouteTarget) -> &str {
8008 match target {
8009 RouteTarget::ToolProvider { module_id }
8010 | RouteTarget::ManagementSurface { module_id }
8011 | RouteTarget::InternalService { module_id, .. } => module_id,
8012 }
8013}
8014
8015fn target_has_required_role(target: &RouteTarget, roles: &[ProviderRole]) -> bool {
8016 roles.iter().any(|role| match (target, role) {
8017 (RouteTarget::ToolProvider { .. }, ProviderRole::ToolProvider { .. }) => true,
8018 (RouteTarget::ManagementSurface { .. }, ProviderRole::ManagementSurface { .. }) => true,
8019 (
8020 RouteTarget::InternalService { service_id, .. },
8021 ProviderRole::InternalService {
8022 service_id: provided,
8023 ..
8024 },
8025 ) => service_id == provided,
8026 _ => false,
8027 })
8028}
8029
8030fn is_routable_role(role: &ProviderRole) -> bool {
8031 matches!(
8032 role,
8033 ProviderRole::ToolProvider { .. }
8034 | ProviderRole::ManagementSurface { .. }
8035 | ProviderRole::InternalService { .. }
8036 )
8037}
8038
8039#[derive(Debug, Clone, Copy, PartialEq, Eq)]
8040enum ControlRequestBodyError {
8041 UnknownOp,
8042 InvalidBody,
8043}
8044
8045#[derive(Debug, Deserialize)]
8046struct ControlOpProbe {
8047 op: String,
8048}
8049
8050const MODULE_PUSH_OPS: &[&str] = &["route.status"];
8053
8054fn is_known_module_push_op(body: &[u8]) -> bool {
8055 serde_json::from_slice::<ControlOpProbe>(body)
8056 .map(|probe| MODULE_PUSH_OPS.contains(&probe.op.as_str()))
8057 .unwrap_or(false)
8058}
8059
8060fn is_known_module_request_op(body: &[u8]) -> bool {
8061 serde_json::from_slice::<ControlOpProbe>(body)
8062 .map(|probe| is_module_to_subc_op(&probe.op))
8063 .unwrap_or(false)
8064}
8065
8066fn is_module_to_subc_op(op: &str) -> bool {
8067 MODULE_TO_SUBC_CONTROL_OPS.contains(&op) || MODULE_TO_SUBC_UNADVERTISED_OPS.contains(&op)
8068}
8069
8070fn log_control_dispatch_arrival(op: &'static str, connection_id: ConnectionId, corr: u64) {
8071 debug!(
8072 op = %op,
8073 connection_id = connection_id.get(),
8074 corr,
8075 "control dispatch"
8076 );
8077}
8078
8079fn log_slow_control_dispatch(
8080 dispatch_started_at: Option<StdInstant>,
8081 op: &'static str,
8082 connection_id: ConnectionId,
8083 corr: u64,
8084 route_open_timings: Option<&RouteOpenTimings>,
8085) {
8086 let Some(dispatch_started_at) = dispatch_started_at else {
8087 return;
8088 };
8089 let elapsed = Instant::from_std(dispatch_started_at).elapsed();
8091 if elapsed >= SLOW_CONTROL_DISPATCH_THRESHOLD {
8092 if let Some(timings) = route_open_timings {
8093 warn!(
8094 op = %op,
8095 connection_id = connection_id.get(),
8096 corr,
8097 elapsed_ms = elapsed.as_millis() as u64,
8098 module_id = ?timings.module_id,
8099 principal = %timings.principal,
8100 admission_ms = timings.admission.as_millis() as u64,
8101 project_root_ms = timings.project_root.as_millis() as u64,
8102 egress_reservation_ms = timings.egress_reservation.as_millis() as u64,
8103 bind_send_ms = timings.bind_send.as_millis() as u64,
8104 bind_ack_wait_ms = timings.bind_ack_wait.as_millis() as u64,
8105 "slow control dispatch"
8106 );
8107 return;
8108 }
8109 warn!(
8110 op = %op,
8111 connection_id = connection_id.get(),
8112 corr,
8113 elapsed_ms = elapsed.as_millis() as u64,
8114 "slow control dispatch"
8115 );
8116 }
8117}
8118
8119#[cfg(test)]
8121type TestCallerLookup =
8122 Arc<dyn Fn(Option<(SocketAddr, SocketAddr)>) -> CallerIdentity + Send + Sync>;
8123
8124fn privileged_operation(request: &ClientControlRequest) -> Option<PrivilegedOperation<'_>> {
8126 match request {
8127 ClientControlRequest::SupervisorRestart { module_id, .. } => {
8128 Some(PrivilegedOperation::Restart { module_id })
8129 }
8130 ClientControlRequest::SupervisorSwap { module_id, .. } => {
8131 Some(PrivilegedOperation::Swap { module_id })
8132 }
8133 ClientControlRequest::SupervisorReload { module_id } => {
8134 Some(PrivilegedOperation::Reload { module_id })
8135 }
8136 ClientControlRequest::SupervisorRescan { preview } => {
8137 if *preview {
8138 None
8139 } else {
8140 Some(PrivilegedOperation::Rescan)
8141 }
8142 }
8143 ClientControlRequest::SupervisorRescanChecked {
8148 preview,
8149 candidate,
8150 expect,
8151 } => (!*preview && candidate.is_none() && expect.is_some())
8152 .then_some(PrivilegedOperation::Rescan),
8153 ClientControlRequest::SupervisorSetEnabled { module_id, enabled } => {
8154 Some(PrivilegedOperation::SetEnabled {
8155 module_id,
8156 enabled: *enabled,
8157 })
8158 }
8159 ClientControlRequest::SupervisorReleaseReserved { module_id } => {
8160 Some(PrivilegedOperation::ReleaseReserved { module_id })
8161 }
8162 ClientControlRequest::ServerDescribe {}
8163 | ClientControlRequest::DaemonShutdownIntent { .. }
8166 | ClientControlRequest::CatalogList { .. }
8167 | ClientControlRequest::RouteOpen { .. }
8168 | ClientControlRequest::RoutePoll { .. }
8169 | ClientControlRequest::SupervisorList {}
8170 | ClientControlRequest::SupervisorHealth {}
8171 | ClientControlRequest::SupervisorHealthProbe { .. }
8172 | ClientControlRequest::SupervisorRoutes { .. }
8173 | ClientControlRequest::SupervisorProvenance { .. }
8174 | ClientControlRequest::SupervisorStderrTail { .. }
8175 | ClientControlRequest::SupervisorTerminals { .. }
8176 | ClientControlRequest::SupervisorSpawnSnapshot {}
8177 | ClientControlRequest::SupervisorSpawnSubscribe { .. } => None,
8178 }
8179}
8180
8181fn client_control_request_op(request: &ClientControlRequest) -> &'static str {
8182 match request {
8183 ClientControlRequest::DaemonShutdownIntent { .. } => ops::DAEMON_SHUTDOWN_INTENT,
8184 ClientControlRequest::ServerDescribe {} => ops::SERVER_DESCRIBE,
8185 ClientControlRequest::SupervisorProvenance { .. } => ops::SUPERVISOR_PROVENANCE,
8186 ClientControlRequest::CatalogList { .. } => ops::CATALOG_LIST,
8187 ClientControlRequest::RouteOpen { .. } => ops::ROUTE_OPEN,
8188 ClientControlRequest::RoutePoll { .. } => ops::ROUTE_POLL,
8189 ClientControlRequest::SupervisorList {} => ops::SUPERVISOR_LIST,
8190 ClientControlRequest::SupervisorSpawnSnapshot {} => ops::SUPERVISOR_SPAWN_SNAPSHOT,
8191 ClientControlRequest::SupervisorSpawnSubscribe { .. } => ops::SUPERVISOR_SPAWN_SUBSCRIBE,
8192 ClientControlRequest::SupervisorRestart { .. } => ops::SUPERVISOR_RESTART,
8193 ClientControlRequest::SupervisorSwap { .. } => ops::SUPERVISOR_SWAP,
8194 ClientControlRequest::SupervisorReload { .. } => ops::SUPERVISOR_RELOAD,
8195 ClientControlRequest::SupervisorRescan { .. } => ops::SUPERVISOR_RESCAN,
8196 ClientControlRequest::SupervisorRescanChecked { .. } => ops::SUPERVISOR_RESCAN_CHECKED,
8197 ClientControlRequest::SupervisorReleaseReserved { .. } => ops::SUPERVISOR_RELEASE_RESERVED,
8198 ClientControlRequest::SupervisorSetEnabled { .. } => ops::SUPERVISOR_SET_ENABLED,
8199 ClientControlRequest::SupervisorHealthProbe { .. } => ops::SUPERVISOR_HEALTH_PROBE,
8200 ClientControlRequest::SupervisorHealth {} => ops::SUPERVISOR_HEALTH,
8201 ClientControlRequest::SupervisorRoutes { .. } => ops::SUPERVISOR_ROUTES,
8202 ClientControlRequest::SupervisorStderrTail { .. } => ops::SUPERVISOR_STDERR_TAIL,
8203 ClientControlRequest::SupervisorTerminals { .. } => ops::SUPERVISOR_TERMINALS,
8204 }
8205}
8206
8207struct RouteOpenCaller {
8215 principal: String,
8219 scope: String,
8222 harness: String,
8225}
8226
8227impl RouteOpenCaller {
8228 fn new(
8229 principal: Option<&Principal>,
8230 scope: Option<&ScopeSelector>,
8231 identity: &BindIdentity,
8232 ) -> Self {
8233 Self {
8234 principal: principal.map_or_else(|| "unattested".to_string(), principal_label),
8235 scope: scope.map_or_else(|| "none".to_string(), route_open_requested_scope_label),
8236 harness: bounded_log_text(&identity.harness),
8237 }
8238 }
8239
8240 fn unknown() -> Self {
8243 Self {
8244 principal: "unknown".to_string(),
8245 scope: "unknown".to_string(),
8246 harness: "unknown".to_string(),
8247 }
8248 }
8249}
8250
8251const MAX_LOGGED_TEXT_CHARS: usize = 64;
8254
8255fn bounded_log_text(value: &str) -> String {
8259 let mut bounded: String = value.chars().take(MAX_LOGGED_TEXT_CHARS).collect();
8260 if value.chars().nth(MAX_LOGGED_TEXT_CHARS).is_some() {
8261 bounded.push('…');
8262 }
8263 format!("{bounded:?}")
8264}
8265
8266fn route_open_scope_label(stamp: &subc_protocol::scope::ScopeStamp) -> String {
8272 use subc_protocol::scope::ScopeKind;
8273 let kind = match stamp.kind {
8274 ScopeKind::Head => "head",
8275 ScopeKind::Worker => "worker",
8276 ScopeKind::Ephemeral => "ephemeral",
8277 ScopeKind::Federated => "federated",
8278 _ => "unknown",
8279 };
8280 format!(
8281 "{kind}/{}/{}@{}",
8282 principal_label(&stamp.owner),
8283 bounded_log_text(&stamp.scope_ref),
8284 stamp.scope_epoch
8285 )
8286}
8287
8288fn route_open_requested_scope_label(selector: &ScopeSelector) -> String {
8295 let epoch = selector
8296 .scope_epoch
8297 .map_or_else(|| "none".to_string(), |epoch| epoch.to_string());
8298 format!(
8299 "requested/{}/{}@{epoch}",
8300 bounded_log_text(&principal_label(&selector.owner)),
8301 bounded_log_text(&selector.scope_ref),
8302 )
8303}
8304
8305fn principal_label(principal: &Principal) -> String {
8306 match principal {
8307 Principal::Reserved { module_id } => format!("reserved:{module_id}"),
8308 Principal::Direct => "direct".to_string(),
8309 other => format!("{other:?}"),
8310 }
8311}
8312
8313fn module_control_request_op(request: &ModuleControlRequestFromModule) -> &'static str {
8314 match request {
8315 ModuleControlRequestFromModule::CatalogUpdate { .. } => MODULE_TO_SUBC_OP_CATALOG_UPDATE,
8316 ModuleControlRequestFromModule::LiveRoots {} => "supervisor.live_roots",
8317 ModuleControlRequestFromModule::ScopeSync { .. } => SCOPE_SYNC_OP,
8318 ModuleControlRequestFromModule::ScopeApply { .. } => SCOPE_APPLY_OP,
8319 ModuleControlRequestFromModule::ScopeDescribe { .. } => SCOPE_DESCRIBE_OP,
8320 ModuleControlRequestFromModule::AgentCommands { .. } => "supervisor.agent_commands",
8321 _ => "unsupported",
8322 }
8323}
8324
8325fn parse_client_control_request(
8326 body: &[u8],
8327) -> Result<ClientControlRequest, (serde_json::Error, ControlRequestBodyError)> {
8328 serde_json::from_slice::<ClientControlRequest>(body).map_err(|err| {
8329 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8330 Ok(probe)
8331 if SUBC_CONTROL_OPS.contains(&probe.op.as_str())
8332 || SUBC_UNADVERTISED_CONTROL_OPS.contains(&probe.op.as_str()) =>
8333 {
8334 ControlRequestBodyError::InvalidBody
8335 }
8336 Ok(_) => ControlRequestBodyError::UnknownOp,
8337 Err(_) => ControlRequestBodyError::InvalidBody,
8338 };
8339 (err, classification)
8340 })
8341}
8342
8343fn parse_module_control_request_from_module(
8344 body: &[u8],
8345) -> Result<ModuleControlRequestFromModule, (serde_json::Error, ControlRequestBodyError)> {
8346 serde_json::from_slice::<ModuleControlRequestFromModule>(body).map_err(|err| {
8347 let classification = match serde_json::from_slice::<ControlOpProbe>(body) {
8348 Ok(probe) if is_module_to_subc_op(&probe.op) => ControlRequestBodyError::InvalidBody,
8349 Ok(_) => ControlRequestBodyError::UnknownOp,
8350 Err(_) => ControlRequestBodyError::InvalidBody,
8351 };
8352 (err, classification)
8353 })
8354}
8355
8356#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
8357enum ProviderRoleKind {
8358 ToolProvider,
8359 PipelineStage,
8360 ManagementSurface,
8361 InternalService,
8362}
8363
8364fn provider_role_kind(role: &ProviderRole) -> ProviderRoleKind {
8365 match role {
8366 ProviderRole::ToolProvider { .. } => ProviderRoleKind::ToolProvider,
8367 ProviderRole::PipelineStage { .. } => ProviderRoleKind::PipelineStage,
8368 ProviderRole::ManagementSurface { .. } => ProviderRoleKind::ManagementSurface,
8369 ProviderRole::InternalService { .. } => ProviderRoleKind::InternalService,
8370 }
8371}
8372
8373fn provider_role_kind_set(roles: &[ProviderRole]) -> BTreeSet<ProviderRoleKind> {
8374 roles.iter().map(provider_role_kind).collect()
8375}
8376
8377const MAX_LOGGED_REFUSED_SCOPE_RECORDS: usize = 8;
8380
8381#[derive(Debug, Default, PartialEq, Eq)]
8383struct ScopeOutcomeCounts {
8384 created: usize,
8385 replaced: usize,
8386 updated: usize,
8387 unchanged: usize,
8388 refused: usize,
8389}
8390
8391impl ScopeOutcomeCounts {
8392 fn of(results: &[ScopeRecordResult]) -> Self {
8393 let mut counts = Self::default();
8394 for result in results {
8395 let slot = match result.outcome {
8396 ScopeRecordOutcome::Created => &mut counts.created,
8397 ScopeRecordOutcome::Replaced => &mut counts.replaced,
8398 ScopeRecordOutcome::Updated => &mut counts.updated,
8399 ScopeRecordOutcome::Unchanged => &mut counts.unchanged,
8400 ScopeRecordOutcome::Refused => &mut counts.refused,
8401 };
8402 *slot += 1;
8403 }
8404 counts
8405 }
8406}
8407
8408#[cfg(test)]
8409mod scope_outcome_count_tests {
8410 use super::*;
8411
8412 fn result(outcome: ScopeRecordOutcome) -> ScopeRecordResult {
8413 ScopeRecordResult {
8414 scope_ref: "r".to_string(),
8415 scope_epoch: 1,
8416 outcome,
8417 code: None,
8418 message: None,
8419 version: None,
8420 parent_state: None,
8421 }
8422 }
8423
8424 #[test]
8427 fn every_outcome_is_counted_in_its_own_field() {
8428 let results = [
8429 result(ScopeRecordOutcome::Created),
8430 result(ScopeRecordOutcome::Created),
8431 result(ScopeRecordOutcome::Replaced),
8432 result(ScopeRecordOutcome::Updated),
8433 result(ScopeRecordOutcome::Unchanged),
8434 result(ScopeRecordOutcome::Refused),
8435 result(ScopeRecordOutcome::Refused),
8436 result(ScopeRecordOutcome::Refused),
8437 ];
8438 assert_eq!(
8439 ScopeOutcomeCounts::of(&results),
8440 ScopeOutcomeCounts {
8441 created: 2,
8442 replaced: 1,
8443 updated: 1,
8444 unchanged: 1,
8445 refused: 3,
8446 }
8447 );
8448 }
8449}
8450
8451fn capability_census_trigger(
8455 old: Option<&CapabilityDeclarations>,
8456 new: Option<&CapabilityDeclarations>,
8457) -> bool {
8458 let old_provides = old
8459 .map(|capabilities| capabilities.provides.iter().collect::<HashSet<_>>())
8460 .unwrap_or_default();
8461 let old_denies = old
8462 .map(|capabilities| capabilities.must_never_reach.iter().collect::<HashSet<_>>())
8463 .unwrap_or_default();
8464 let new = new.cloned().unwrap_or(CapabilityDeclarations {
8465 provides: Vec::new(),
8466 requires: Vec::new(),
8467 must_never_reach: Vec::new(),
8468 });
8469
8470 new.provides
8471 .iter()
8472 .any(|capability| !old_provides.contains(capability))
8473 || new
8474 .must_never_reach
8475 .iter()
8476 .any(|capability| !old_denies.contains(capability))
8477}
8478
8479fn denied_capability<'a>(
8482 opening_manifest: &'a ModuleManifest,
8483 target_manifest: &ModuleManifest,
8484) -> Option<&'a str> {
8485 let opening_capabilities = opening_manifest.capabilities.as_ref()?;
8486 let target_capabilities = target_manifest.capabilities.as_ref()?;
8487 opening_capabilities
8488 .must_never_reach
8489 .iter()
8490 .find(|denied| {
8491 target_capabilities
8492 .provides
8493 .iter()
8494 .any(|provided| provided == *denied)
8495 })
8496 .map(String::as_str)
8497}
8498
8499fn catalog_update_frozen_field_message(
8500 registered: &ModuleManifest,
8501 provides: &[ProviderRole],
8502) -> Option<String> {
8503 let old_has_provides = !registered.provides.is_empty();
8504 let new_has_provides = !provides.is_empty();
8505 if old_has_provides != new_has_provides {
8506 return Some(format!(
8507 "catalog.update cannot change module '{}' between supervision-only and routable; routability is fixed at HELLO",
8508 registered.module_id
8509 ));
8510 }
8511
8512 if provider_role_kind_set(®istered.provides) != provider_role_kind_set(provides) {
8513 return Some(format!(
8514 "catalog.update cannot change provider role kinds for module '{}'; role kinds are fixed at HELLO",
8515 registered.module_id
8516 ));
8517 }
8518
8519 let registered_concurrency = manifest_concurrency(registered);
8520 let mut candidate = registered.clone();
8521 candidate.provides = provides.to_vec();
8522 let candidate_concurrency = manifest_concurrency(&candidate);
8523 if candidate_concurrency != registered_concurrency {
8524 return Some(format!(
8525 "catalog.update cannot change module '{}' concurrency from {:?} to {:?}; concurrency is fixed at HELLO",
8526 registered.module_id, registered_concurrency, candidate_concurrency
8527 ));
8528 }
8529
8530 None
8533}
8534
8535fn manifest_provides_routable_role(manifest: &ModuleManifest) -> bool {
8536 manifest.provides.iter().any(is_routable_role)
8537}
8538
8539fn manifest_concurrency(manifest: &ModuleManifest) -> Concurrency {
8545 manifest
8546 .provides
8547 .iter()
8548 .find_map(|provider| match provider {
8549 ProviderRole::ToolProvider { concurrency, .. }
8550 | ProviderRole::ManagementSurface { concurrency, .. } => Some(concurrency.clone()),
8551 ProviderRole::PipelineStage { .. } | ProviderRole::InternalService { .. } => None,
8552 })
8553 .unwrap_or(Concurrency::ModuleManaged)
8554}
8555
8556fn manifest_concurrency_was_defaulted(raw_hello: &[u8], manifest: &ModuleManifest) -> bool {
8563 let has_management_surface = manifest
8564 .provides
8565 .iter()
8566 .any(|provider| matches!(provider, ProviderRole::ManagementSurface { .. }));
8567 if !has_management_surface {
8568 return false;
8569 }
8570 let Ok(raw) = serde_json::from_slice::<serde_json::Value>(raw_hello) else {
8571 return false;
8572 };
8573 let Some(provides) = raw
8574 .get("manifest")
8575 .and_then(|manifest| manifest.get("provides"))
8576 .and_then(serde_json::Value::as_array)
8577 else {
8578 return false;
8579 };
8580 provides.iter().any(|role| {
8585 role.get("role").and_then(serde_json::Value::as_str) == Some("management_surface")
8586 && role.get("concurrency").is_none()
8587 })
8588}
8589
8590fn negotiate_version(peer_version: u8) -> Result<u8, String> {
8591 if peer_version != PROTOCOL_VERSION {
8592 return Err(format!(
8593 "protocol_ver {peer_version} is unsupported; this daemon requires exactly {PROTOCOL_VERSION}"
8594 ));
8595 }
8596 Ok(PROTOCOL_VERSION)
8597}
8598
8599fn pong(frame: &Frame) -> Result<Frame, RouterError> {
8600 Frame::build_with_version(
8601 response_version(frame),
8602 FrameType::Pong,
8603 frame.header.flags,
8604 0,
8605 0,
8606 frame.header.corr,
8607 Vec::new(),
8608 )
8609 .map_err(RouterError::FrameBuild)
8610}
8611
8612pub(crate) fn control_error_frame(
8613 frame: &Frame,
8614 code: &'static str,
8615 message: impl Into<String>,
8616) -> Result<Frame, RouterError> {
8617 control_error_body_frame(
8618 frame,
8619 ErrorBody {
8620 code: code.to_string(),
8621 message: message.into(),
8622 detail: None,
8623 },
8624 )
8625}
8626
8627fn control_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8635 control_error_frame_with_flags(frame, error, daemon_error_flags())
8636}
8637
8638fn control_relayed_error_body_frame(frame: &Frame, error: ErrorBody) -> Result<Frame, RouterError> {
8644 control_error_frame_with_flags(frame, error, control_flags())
8645}
8646
8647fn control_error_frame_with_flags(
8648 frame: &Frame,
8649 error: ErrorBody,
8650 flags: Flags,
8651) -> Result<Frame, RouterError> {
8652 let body = serde_json::to_vec(&error).map_err(|err| {
8653 RouterError::backend(
8654 0,
8655 frame.header.corr,
8656 format!("failed to encode control ERROR: {err}"),
8657 )
8658 })?;
8659
8660 Frame::build_with_version(
8661 response_version(frame),
8662 FrameType::Error,
8663 flags,
8664 0,
8665 0,
8666 frame.header.corr,
8667 body,
8668 )
8669 .map_err(RouterError::FrameBuild)
8670}
8671
8672pub(crate) fn control_response_body_frame<T: Serialize>(
8673 frame: &Frame,
8674 reply: &T,
8675 label: &'static str,
8676) -> Result<Frame, RouterError> {
8677 let body = serde_json::to_vec(reply).map_err(|err| {
8678 RouterError::backend(
8679 0,
8680 frame.header.corr,
8681 format!("failed to encode {label}: {err}"),
8682 )
8683 })?;
8684
8685 Frame::build_with_version(
8686 response_version(frame),
8687 FrameType::Response,
8688 control_flags(),
8689 0,
8690 0,
8691 frame.header.corr,
8692 body,
8693 )
8694 .map_err(RouterError::FrameBuild)
8695}
8696
8697fn forwarding_error_code(err: &ForwardingError) -> &'static str {
8727 match err {
8728 ForwardingError::ConnectionRoleConflict { .. } => "invalid_request",
8729 ForwardingError::NoModuleConnection => "target_unavailable",
8730 ForwardingError::ModuleReloading { .. } => "module_reloading",
8731 ForwardingError::ClientRouteChannelExhausted { .. }
8732 | ForwardingError::ModuleRouteChannelExhausted { .. } => "route_limit",
8733 ForwardingError::StaleModuleEndpoint
8734 | ForwardingError::UnknownReservation { .. }
8735 | ForwardingError::ConnectionClosing { .. }
8736 | ForwardingError::ClientEgressClosed { .. }
8737 | ForwardingError::ModuleEgressUnavailable { .. } => "target_unavailable",
8738 ForwardingError::CandidateSlotOccupied { .. } => "duplicate_module_id",
8741 ForwardingError::RelayCorrelationExhausted
8742 | ForwardingError::RouteOpenBuild(_)
8743 | ForwardingError::Poisoned => "forwarding_error",
8744 }
8745}
8746
8747fn response_version(frame: &Frame) -> u8 {
8748 if (MIN_SUPPORTED_VERSION..=PROTOCOL_VERSION).contains(&frame.header.ver) {
8749 frame.header.ver
8750 } else {
8751 PROTOCOL_VERSION
8752 }
8753}
8754
8755fn control_flags() -> Flags {
8756 Flags::new(false, Priority::Passive, false)
8757}
8758
8759fn send_goodbye_target_best_effort(
8764 counters: &DaemonCounters,
8765 target: &GoodbyeTarget,
8766 context: &'static str,
8767) {
8768 let Ok(frame) = Frame::build_with_version(
8769 target.negotiated_ver,
8770 FrameType::Goodbye,
8771 control_flags(),
8772 target.channel,
8773 target.epoch,
8774 0,
8775 Vec::new(),
8776 ) else {
8777 return;
8778 };
8779 crate::forwarding::send_module_route_goodbye(
8780 counters,
8781 &target.sink,
8782 frame,
8783 target.module_id.as_deref(),
8784 context,
8785 );
8786}
8787
8788pub(crate) fn send_route_control_pushes(
8789 forwarding: &ForwardingTable,
8790 routes: Vec<EndpointRoute>,
8791 push: ClientControlPush,
8792) {
8793 let mut targets: Vec<(GoodbyeTarget, Vec<u16>)> = Vec::new();
8794 for route in routes {
8795 let target = route.goodbye_target;
8796 if let Some((existing, channels)) = targets
8797 .iter_mut()
8798 .find(|(existing, _)| existing.connection_id == target.connection_id)
8799 {
8800 debug_assert_eq!(
8801 existing.negotiated_ver, target.negotiated_ver,
8802 "one connection cannot negotiate multiple frame versions"
8803 );
8804 if !channels.contains(&target.channel) {
8805 channels.push(target.channel);
8806 }
8807 continue;
8808 }
8809 let channel = target.channel;
8810 targets.push((target, vec![channel]));
8811 }
8812 for (target, mut channels) in targets {
8813 channels.sort_unstable();
8814 let mut push = push.clone();
8815 match &mut push {
8816 ClientControlPush::RouteClosing {
8817 channels: covered, ..
8818 }
8819 | ClientControlPush::RouteClosed {
8820 channels: covered, ..
8821 } => *covered = channels,
8822 }
8823 let body = match serde_json::to_vec(&push) {
8824 Ok(body) => body,
8825 Err(err) => {
8826 warn!(error = %err, "failed to serialize route lifecycle control PUSH");
8827 continue;
8828 }
8829 };
8830 let frame = match Frame::build_with_version(
8831 target.negotiated_ver,
8832 FrameType::Push,
8833 control_flags(),
8834 0,
8835 0,
8836 0,
8837 body.clone(),
8838 ) {
8839 Ok(frame) => frame,
8840 Err(err) => {
8841 warn!(
8842 route_channel = target.channel,
8843 error = %err,
8844 "failed to build route lifecycle control PUSH frame"
8845 );
8846 continue;
8847 }
8848 };
8849 if let Err(err) = target.sink.try_send(frame) {
8850 if target.close_on_delivery_failure() {
8851 warn!(
8852 target_connection_id = target.connection_id.get(),
8853 route_channel = target.channel,
8854 error = %err,
8855 "route lifecycle control PUSH was not delivered to client; closing target connection"
8856 );
8857 let _ = forwarding.escalate_client_delivery_failure(
8858 target.connection_id,
8859 target.channel,
8860 target.epoch,
8861 CloseReason::new(
8862 "route_lifecycle_push_delivery_failed",
8863 format!(
8864 "failed to enqueue route lifecycle control PUSH for channel {}: {err}",
8865 target.channel
8866 ),
8867 ),
8868 crate::forwarding::UndeliveredFrame {
8869 module_id: target.module_id.as_deref(),
8870 sink: &target.sink,
8871 },
8872 );
8873 }
8874 }
8875 }
8876}
8877
8878#[cfg(test)]
8879mod tests {
8880 include!("control/agent_commands_tests.rs");
8881 include!("control/resource_revision_tests.rs");
8882 #[tokio::test]
8883 async fn route_busy_wait_attribution_resolves_targets_instead_of_trusting_selectors() {
8884 let registry = Arc::new(Registry::default());
8885 let handler = ControlHandler::new(registry);
8886 assert_eq!(
8887 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8888 None
8889 );
8890 let (ctx, mut rx) = route_ctx(ConnectionId::new(901));
8891 hello_via_sink(
8892 &handler,
8893 &ctx,
8894 &mut rx,
8895 hello_frame("daemon", PROTOCOL_VERSION, 1),
8896 )
8897 .await;
8898 assert_eq!(
8899 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("daemon")),
8900 Some("daemon".into())
8901 );
8902 assert_eq!(
8903 handler.resolve_reader_wait_target(ops::SUPERVISOR_HEALTH_PROBE, Some("missing")),
8904 None
8905 );
8906 assert_eq!(
8907 handler.resolve_reader_wait_target(ops::SUPERVISOR_TERMINALS, Some("daemon")),
8908 None,
8909 "a registered module is not necessarily supervised"
8910 );
8911 assert_eq!(
8912 handler.resolve_reader_wait_target(ops::SUPERVISOR_PROVENANCE, None),
8913 None,
8914 "aggregate provenance has no single target"
8915 );
8916 assert_eq!(
8917 handler.resolve_reader_wait_target(ops::APP_OPEN, Some("daemon")),
8918 None
8919 );
8920 }
8921
8922 mod peer_manifest {
8923 include!("peer_manifest_tests.rs");
8924 }
8925 mod activity_read {
8926 include!("activity_read_tests.rs");
8927 }
8928
8929 mod lifecycle_activity {
8930 include!("control_lifecycle_tests.rs");
8931 }
8932 #[cfg(unix)]
8933 #[tokio::test]
8934 async fn rescan_health_only_is_live_but_launch_edits_need_reload() {
8935 let dir = cortexkit_test_support::ScratchDir::new("rescan-live-health");
8936 let path = dir.join("subc.jsonc");
8937 std::fs::write(&path, serde_json::json!({"version":1,"modules":{"stock":{
8938 "program":"/bin/sleep","args":["60"],"protocol":"none",
8939 "env":{"XDG_DATA_HOME":dir.path(),"XDG_RUNTIME_DIR":dir.path(),"XDG_CONFIG_HOME":dir.path()}
8940 }}}).to_string()).unwrap();
8941 let mut configured = crate::daemon_config::load(&path)
8942 .unwrap()
8943 .unwrap()
8944 .modules
8945 .pop()
8946 .unwrap();
8947 let registry = std::sync::Arc::new(crate::Registry::default());
8948 let handle = crate::SupervisorHandle::new();
8949 let supervisor = crate::Supervisor::new(registry.clone(), crate::RestartPolicy::default())
8950 .with_handle(handle.clone());
8951 let module = supervisor
8952 .supervise_configured_with_health(
8953 configured.module_spec(),
8954 true,
8955 configured.health.clone(),
8956 None,
8957 configured.restart,
8958 )
8959 .unwrap();
8960 let handler = super::ControlHandler::new(registry).with_supervisor(handle);
8961 let before = module.status().unwrap().pid;
8962 configured.health.http = Some("http://127.0.0.1:1/healthz".into());
8963 configured.health.cadence = std::time::Duration::from_secs(3600);
8964 let health_only = handler
8965 .reconcile_supervised_modules(
8966 &supervisor,
8967 vec![configured.clone()],
8968 false,
8969 Principal::Direct,
8970 )
8971 .await
8972 .unwrap();
8973 assert!(
8974 health_only.0.changed_pending_reload.is_empty(),
8975 "health policy is already applied live"
8976 );
8977 assert_eq!(module.status().unwrap().pid, before);
8978 assert_eq!(
8979 module.configuration().unwrap().1.http,
8980 configured.health.http
8981 );
8982 configured.args = vec!["61".into()];
8983 let launch = handler
8984 .reconcile_supervised_modules(&supervisor, vec![configured], false, Principal::Direct)
8985 .await
8986 .unwrap();
8987 assert_eq!(launch.0.changed_pending_reload, ["stock"]);
8988 assert_eq!(
8989 module.status().unwrap().pid,
8990 before,
8991 "a launch edit is stored until reload"
8992 );
8993 module.drain().await.unwrap();
8994 }
8995 use cortexkit_test_support::ScratchDir;
8996 use std::{
8997 collections::BTreeMap,
8998 fmt,
8999 path::PathBuf,
9000 sync::{Arc, Mutex},
9001 time::Duration,
9002 };
9003
9004 use serde_json::{json, Value};
9005 use subc_protocol::{
9006 manifest::{
9007 Concurrency, ExecutionMode, IdentityScope, ManagementOperation,
9008 ManagementOperationKind, ObservabilityKind, ObservabilitySurface, ProviderRole, Tool,
9009 },
9010 session::HealthStatus,
9011 FrameType,
9012 };
9013
9014 use super::*;
9015 use crate::{
9016 forwarding::{DataRoute, DataRouteState},
9017 registry::ChannelState,
9018 router::FrameSink,
9019 stderr_tail::DEFAULT_MAX_LINE_BYTES,
9020 supervise::{ModuleSpec, ModuleState, RestartPolicy, Supervisor, SupervisorHandle},
9021 RouteCtx, Router,
9022 };
9023 use tokio::{
9024 sync::mpsc,
9025 time::{sleep, Instant},
9026 };
9027 use tracing::{
9028 field::{Field, Visit},
9029 Event, Subscriber,
9030 };
9031 use tracing_subscriber::{layer::Context, prelude::*, Layer};
9032
9033 fn fake_aft_stub_path() -> PathBuf {
9052 let mut path = std::env::current_exe().expect("current_exe available in tests");
9053 path.pop(); path.pop(); path.push(if cfg!(windows) {
9056 "fake-aft-stub.exe"
9057 } else {
9058 "fake-aft-stub"
9059 });
9060 assert!(
9061 path.exists(),
9062 "fake-aft-stub not built at {}: run `cargo test -p subc-core` (which builds \
9063 [[bin]] targets) rather than `cargo test -p subc-core --lib` (which does not)",
9064 path.display()
9065 );
9066 path
9067 }
9068
9069 fn client_retries(code: &str) -> bool {
9074 subc_protocol::error_codes::is_retryable_route_open(code)
9075 }
9076
9077 #[test]
9088 fn retryability_of_forwarding_codes_matches_the_failure() {
9089 let transient = [
9092 ForwardingError::NoModuleConnection,
9093 ForwardingError::ModuleReloading {
9094 module_id: "m".into(),
9095 },
9096 ForwardingError::StaleModuleEndpoint,
9097 ForwardingError::UnknownReservation {
9098 client_channel: 1,
9099 module_channel: 1,
9100 },
9101 ForwardingError::ConnectionClosing {
9102 connection_id: ConnectionId::new(1),
9103 },
9104 ForwardingError::ClientEgressClosed {
9105 connection_id: ConnectionId::new(1),
9106 },
9107 ForwardingError::ModuleEgressUnavailable {
9108 connection_id: ConnectionId::new(1),
9109 },
9110 ];
9111 for err in transient {
9112 let code = forwarding_error_code(&err);
9113 assert!(
9114 client_retries(code),
9115 "{err:?} is transient but publishes {code:?}, which clients treat as permanent"
9116 );
9117 }
9118
9119 let permanent = [
9124 ForwardingError::ConnectionRoleConflict {
9125 connection_id: ConnectionId::new(1),
9126 },
9127 ForwardingError::ClientRouteChannelExhausted {
9128 connection_id: ConnectionId::new(1),
9129 },
9130 ForwardingError::ModuleRouteChannelExhausted {
9131 endpoint: ModuleEndpointId {
9132 connection_id: ConnectionId::new(1),
9133 generation: 1,
9134 },
9135 },
9136 ForwardingError::RelayCorrelationExhausted,
9137 ForwardingError::RouteOpenBuild("x".into()),
9138 ForwardingError::Poisoned,
9139 ];
9140 for err in permanent {
9141 let code = forwarding_error_code(&err);
9142 assert!(
9143 !client_retries(code),
9144 "{err:?} cannot be fixed by retrying but publishes {code:?}, which clients retry"
9145 );
9146 }
9147 }
9148
9149 #[tokio::test]
9177 async fn an_unattested_caller_is_never_stamped_as_a_supervised_module() {
9178 let handler = ControlHandler::default();
9179 let frame =
9180 Frame::build(FrameType::Request, control_flags(), 0, 0, 900, Vec::new()).unwrap();
9181
9182 let stamped = handler.route_open_principal(&frame, None).unwrap().unwrap();
9186 assert_eq!(
9187 stamped,
9188 Principal::Direct,
9189 "a caller that proved nothing must not be stamped as a supervised module"
9190 );
9191
9192 let forged = handler
9197 .route_open_principal(
9198 &frame,
9199 Some(ConsumerIdentity {
9200 module_id: "aft".to_string(),
9201 launch_nonce: "not-a-real-nonce".to_string(),
9202 }),
9203 )
9204 .unwrap();
9205 let refusal = forged.expect_err("an unmatched launch nonce must not yield a principal");
9206 assert_eq!(parse_error(&refusal)["code"], "bad_consumer_identity");
9207 }
9208
9209 #[test]
9226 fn a_wire_body_actually_yields_the_consumer_identity_the_daemon_stamps_from() {
9227 let body = br#"{"op":"route.open","target":{"kind":"tool_provider","module_id":"m"},"identity":{"session":"s","project_root":"/p","harness":"h"},"consumer_identity":{"module_id":"aft","launch_nonce":"n"}}"#;
9228 let parsed: ClientControlRequest = serde_json::from_slice(body).unwrap();
9229 let ClientControlRequest::RouteOpen {
9230 consumer_identity, ..
9231 } = parsed
9232 else {
9233 panic!("route.open body must parse as RouteOpen");
9234 };
9235 assert_eq!(
9236 consumer_identity,
9237 Some(ConsumerIdentity {
9238 module_id: "aft".to_string(),
9239 launch_nonce: "n".to_string(),
9240 }),
9241 "the wire field name must reach the value route_open_principal reads"
9242 );
9243 }
9244
9245 fn manifest(module_id: &str, protocol_ver: u8) -> ModuleManifest {
9246 ModuleManifest::builder(module_id, "0.1.0")
9247 .protocol_ver(protocol_ver)
9248 .provides(vec![ProviderRole::ToolProvider {
9249 tools: vec![Tool {
9250 name: "read".to_string(),
9251 description: None,
9252 execution_mode: ExecutionMode::Pure,
9253 schema: json!({"type": "object"}),
9254 }],
9255 identity_scope: vec![IdentityScope::Project, IdentityScope::Session],
9256 concurrency: Concurrency::ModuleManaged,
9257 emits_push: true,
9258 sub_supervises: true,
9259 }])
9260 .build()
9261 }
9262
9263 fn hello_frame(module_id: &str, protocol_ver: u8, corr: u64) -> Frame {
9264 hello_frame_with_control_ops(module_id, protocol_ver, corr, None)
9265 }
9266
9267 fn hello_frame_with_control_ops(
9268 module_id: &str,
9269 protocol_ver: u8,
9270 corr: u64,
9271 control_ops: Option<Vec<String>>,
9272 ) -> Frame {
9273 hello_frame_full(module_id, protocol_ver, corr, control_ops, None)
9274 }
9275
9276 fn hello_frame_with_nonce(
9277 module_id: &str,
9278 protocol_ver: u8,
9279 corr: u64,
9280 launch_nonce: Option<&str>,
9281 ) -> Frame {
9282 hello_frame_full(
9283 module_id,
9284 protocol_ver,
9285 corr,
9286 None,
9287 launch_nonce.map(ToOwned::to_owned),
9288 )
9289 }
9290
9291 fn hello_frame_full(
9292 module_id: &str,
9293 protocol_ver: u8,
9294 corr: u64,
9295 control_ops: Option<Vec<String>>,
9296 launch_nonce: Option<String>,
9297 ) -> Frame {
9298 let body = serde_json::to_vec(&ModuleHelloBody {
9299 manifest: manifest(module_id, protocol_ver),
9300 protocol_ver,
9301 control_ops,
9302 launch_nonce,
9303 })
9304 .unwrap();
9305 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9306 }
9307
9308 fn hello_frame_with_rich_ui(
9309 module_id: &str,
9310 protocol_ver: u8,
9311 corr: u64,
9312 launch_nonce: Option<&str>,
9313 rich_ui: Value,
9314 ) -> Frame {
9315 let mut body = serde_json::to_value(ModuleHelloBody {
9316 manifest: manifest(module_id, protocol_ver),
9317 protocol_ver,
9318 control_ops: None,
9319 launch_nonce: launch_nonce.map(str::to_string),
9320 })
9321 .unwrap();
9322 body["manifest"]["rich_ui"] = rich_ui;
9323 Frame::build(
9324 FrameType::Hello,
9325 control_flags(),
9326 0,
9327 0,
9328 corr,
9329 serde_json::to_vec(&body).unwrap(),
9330 )
9331 .unwrap()
9332 }
9333
9334 fn non_routable_hello_frame_with_control_ops(
9335 module_id: &str,
9336 corr: u64,
9337 control_ops: Option<Vec<String>>,
9338 ) -> Frame {
9339 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
9340 manifest.provides.clear();
9341 let body = serde_json::to_vec(&ModuleHelloBody {
9342 manifest,
9343 protocol_ver: PROTOCOL_VERSION,
9344 control_ops,
9345 launch_nonce: None,
9346 })
9347 .unwrap();
9348 Frame::build(FrameType::Hello, control_flags(), 0, 0, corr, body).unwrap()
9349 }
9350
9351 fn capability_grammar_hello_frame(
9352 capabilities: Value,
9353 runtime_computed: Option<Value>,
9354 corr: u64,
9355 ) -> Frame {
9356 let mut body = serde_json::to_value(ModuleHelloBody {
9357 manifest: manifest("capability-grammar-test", PROTOCOL_VERSION),
9358 protocol_ver: PROTOCOL_VERSION,
9359 control_ops: None,
9360 launch_nonce: None,
9361 })
9362 .expect("HELLO body serializes");
9363 body["manifest"]["capabilities"] = capabilities;
9364 if let Some(runtime_computed) = runtime_computed {
9365 body["runtime_computed"] = runtime_computed;
9366 }
9367 Frame::build(
9368 FrameType::Hello,
9369 control_flags(),
9370 0,
9371 0,
9372 corr,
9373 serde_json::to_vec(&body).expect("HELLO body reserializes"),
9374 )
9375 .expect("HELLO frame builds")
9376 }
9377
9378 fn channel_request(channel: u16, corr: u64) -> Frame {
9379 Frame::build(
9380 FrameType::Request,
9381 Flags::new(true, Priority::Interactive, false),
9382 channel,
9383 0,
9384 corr,
9385 b"opaque".to_vec(),
9386 )
9387 .unwrap()
9388 }
9389
9390 fn route_ctx(
9391 connection_id: ConnectionId,
9392 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
9393 let (tx, rx) = mpsc::channel(8);
9394 (
9395 RouteCtx {
9396 connection_id,
9397 egress: FrameSink::new(tx),
9398 },
9399 rx,
9400 )
9401 }
9402
9403 fn parse_ack(frame: &Frame) -> ModuleHelloAckBody {
9404 serde_json::from_slice(&frame.body).unwrap()
9405 }
9406
9407 async fn hello_via_sink(
9412 handler: &ControlHandler,
9413 ctx: &RouteCtx,
9414 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
9415 hello: Frame,
9416 ) -> Frame {
9417 let replies = handler.handle_control_frame(ctx, hello).await.unwrap();
9418 assert!(
9419 replies.is_empty(),
9420 "a registered HELLO replies with nothing; its ack is already queued: {replies:?}"
9421 );
9422 let ack = rx
9423 .try_recv()
9424 .expect("HELLO_ACK is queued on the module sink")
9425 .frame;
9426 assert_eq!(ack.header.ty, FrameType::HelloAck);
9427 ack
9428 }
9429
9430 fn parse_error(frame: &Frame) -> Value {
9431 serde_json::from_slice(&frame.body).unwrap()
9432 }
9433
9434 fn parse_route_poll(frame: &Frame) -> ClientControlResponse {
9435 serde_json::from_slice(&frame.body).unwrap()
9436 }
9437
9438 fn route_poll_frame(corr: u64, kind: PollKind, route_channel: u16) -> Frame {
9439 let body = serde_json::to_vec(&ClientControlRequest::RoutePoll {
9440 route_channel,
9441 route_epoch: 0,
9442 kind,
9443 })
9444 .unwrap();
9445 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9446 }
9447
9448 fn supervisor_health_probe_frame(corr: u64, module_id: &str) -> Frame {
9449 let body = serde_json::to_vec(&ClientControlRequest::SupervisorHealthProbe {
9450 module_id: module_id.to_string(),
9451 })
9452 .unwrap();
9453 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9454 }
9455
9456 fn route_open_frame(corr: u64, module_id: &str, project_root: ScratchDir) -> Frame {
9457 route_open_frame_with_consumer_capabilities(corr, module_id, project_root, None)
9458 }
9459
9460 fn legacy_route_open_frame(corr: u64, module_id: &str, root: ScratchDir) -> Frame {
9462 let path = ProjectRootId::from_path_allowing_missing(root.path()).unwrap();
9463 let mut frame = route_open_frame(corr, module_id, root);
9464 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
9465 body["identity"]["project_root"] = serde_json::json!(path.as_path());
9466 frame.body = serde_json::to_vec(&body).unwrap();
9467 frame
9468 }
9469
9470 fn route_open_frame_with_consumer_capabilities(
9471 corr: u64,
9472 module_id: &str,
9473 _project_root: ScratchDir,
9474 consumer_capabilities: Option<Vec<String>>,
9475 ) -> Frame {
9476 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9477 busy_refusals: None,
9478 target: RouteTarget::ToolProvider {
9479 module_id: module_id.to_string(),
9480 },
9481 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
9482 consumer_identity: None,
9483 consumer_capabilities,
9484 role_versions: None,
9485 admission_facts: None,
9486 scope: None,
9487 })
9488 .unwrap();
9489 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9490 }
9491
9492 fn route_open_frame_with_role_versions(
9493 corr: u64,
9494 module_id: &str,
9495 _project_root: ScratchDir,
9496 role_versions: Option<BTreeMap<String, String>>,
9497 ) -> Frame {
9498 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9499 busy_refusals: None,
9500 target: RouteTarget::ToolProvider {
9501 module_id: module_id.to_string(),
9502 },
9503 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9504 consumer_identity: None,
9505 consumer_capabilities: None,
9506 role_versions,
9507 admission_facts: None,
9508 scope: None,
9509 })
9510 .unwrap();
9511 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9512 }
9513
9514 fn role_versions(entries: &[(&str, &str)]) -> BTreeMap<String, String> {
9515 entries
9516 .iter()
9517 .map(|(role, version)| (role.to_string(), version.to_string()))
9518 .collect()
9519 }
9520
9521 fn route_open_frame_with_admission_facts(
9522 corr: u64,
9523 module_id: &str,
9524 _project_root: ScratchDir,
9525 consumer_identity: Option<subc_control::ConsumerIdentity>,
9526 facts: Option<Value>,
9527 ) -> Frame {
9528 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
9529 busy_refusals: None,
9530 target: RouteTarget::ToolProvider {
9531 module_id: module_id.to_string(),
9532 },
9533 identity: BindIdentity::new("unit".to_string(), format!("session-{corr}")),
9534 consumer_identity,
9535 consumer_capabilities: None,
9536 role_versions: None,
9537 admission_facts: facts,
9538 scope: None,
9539 })
9540 .unwrap();
9541 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
9542 }
9543
9544 #[derive(Clone, Default)]
9545 struct EventCapture {
9546 events: Arc<Mutex<Vec<CapturedEvent>>>,
9547 }
9548
9549 #[derive(Clone, Debug)]
9550 struct CapturedEvent {
9551 target: String,
9552 level: tracing::Level,
9553 fields: BTreeMap<String, String>,
9554 }
9555
9556 impl EventCapture {
9557 fn events(&self) -> Vec<CapturedEvent> {
9558 self.events.lock().unwrap().clone()
9559 }
9560 }
9561
9562 impl<S> Layer<S> for EventCapture
9563 where
9564 S: Subscriber,
9565 {
9566 fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
9567 let mut visitor = EventFieldVisitor::default();
9568 event.record(&mut visitor);
9569 self.events.lock().unwrap().push(CapturedEvent {
9570 target: event.metadata().target().to_string(),
9571 level: *event.metadata().level(),
9572 fields: visitor.fields,
9573 });
9574 }
9575 }
9576
9577 #[derive(Default)]
9578 struct EventFieldVisitor {
9579 fields: BTreeMap<String, String>,
9580 }
9581
9582 impl Visit for EventFieldVisitor {
9583 fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
9584 self.fields
9585 .insert(field.name().to_string(), format!("{value:?}"));
9586 }
9587 }
9588
9589 fn health_response(corr: u64, status: HealthStatus) -> Frame {
9590 let body = serde_json::to_vec(&ModuleControlResponse::HealthCheck {
9591 status,
9592 detail: Some("warming".to_string()),
9593 metrics: Some(json!({"queue_depth": 3})),
9594 })
9595 .unwrap();
9596 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9597 }
9598
9599 fn route_bind_ack(corr: u64) -> Frame {
9600 let body = serde_json::to_vec(&ModuleControlResponse::RouteBindAck {}).unwrap();
9601 Frame::build(FrameType::Response, control_flags(), 0, 0, corr, body).unwrap()
9602 }
9603
9604 fn unique_project_root(label: &str) -> ScratchDir {
9605 ScratchDir::new(label)
9606 }
9607
9608 fn assert_route_poll_liveness(frame: &Frame, expected_live: bool) {
9609 match parse_route_poll(frame) {
9610 ClientControlResponse::RoutePoll {
9611 status: None,
9612 live: Some(live),
9613 ..
9614 } => assert_eq!(live, expected_live),
9615 other => panic!("unexpected route.poll response: {other:?}"),
9616 }
9617 }
9618
9619 fn bind_liveness_route(
9620 registry: &Registry,
9621 forwarding: &ForwardingTable,
9622 module_id: &str,
9623 ) -> (RouteCtx, u16, u32) {
9624 let module_connection = ConnectionId::new(101);
9625 let client_connection = ConnectionId::new(202);
9626 let registration = registry
9627 .register_with_control_ops(
9628 manifest(module_id, PROTOCOL_VERSION),
9629 PROTOCOL_VERSION,
9630 module_connection,
9631 module_baseline_control_ops(),
9632 )
9633 .unwrap();
9634 let (module_tx, _module_rx) = mpsc::channel(8);
9635 let endpoint = forwarding
9636 .register_module_connection(
9637 module_connection,
9638 module_id.to_string(),
9639 PROTOCOL_VERSION,
9640 manifest_concurrency(®istration.manifest),
9641 FrameSink::new(module_tx),
9642 )
9643 .unwrap();
9644 let (client_ctx, _client_rx) = route_ctx(client_connection);
9645 let pending = forwarding
9646 .begin_route_bind_relay_for_test(
9647 client_connection,
9648 client_ctx.egress.clone(),
9649 1,
9650 module_id,
9651 )
9652 .unwrap();
9653 assert_eq!(pending.endpoint, endpoint);
9654 let route_channel = pending.client_channel;
9655 let route_epoch = pending.client_epoch;
9656 forwarding
9657 .complete_pending_relay(
9658 module_connection,
9659 pending.corr,
9660 RouteBindRelayOutcome::Accepted,
9661 )
9662 .unwrap();
9663 (client_ctx, route_channel, route_epoch)
9664 }
9665
9666 struct FakeProcessLiveness {
9667 live: Option<bool>,
9668 }
9669
9670 impl ModuleProcessLiveness for FakeProcessLiveness {
9671 fn process_live(&self, _module_id: &str) -> Option<bool> {
9672 self.live
9673 }
9674 }
9675
9676 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9677 async fn supervisor_stderr_tail_converts_a_real_truncated_ring_entry_to_prefix_only_wire_data()
9678 {
9679 let registry = Arc::new(Registry::default());
9680 let supervisor_handle = SupervisorHandle::new();
9681 let supervisor = Supervisor::new_for_test(
9682 Arc::clone(®istry),
9683 RestartPolicy::new(1, Duration::from_millis(10)),
9684 )
9685 .with_handle(supervisor_handle.clone());
9686 let source_line = format!("config error: {}", "x".repeat(DEFAULT_MAX_LINE_BYTES));
9687 let module = supervisor
9688 .spawn(ModuleSpec {
9689 carry_containment: false,
9690 module_id: "stderr-tail-wire".to_string(),
9691 program: fake_aft_stub_path(),
9692 args: Vec::new(),
9693 env: vec![
9694 ("FAKE_AFT_STDERR_LINE".to_string(), source_line.clone()),
9695 ("FAKE_AFT_EXIT_CODE".to_string(), "1".to_string()),
9696 ],
9697 reserved: false,
9698 reserved_prefixes: Vec::new(),
9699 protocol: ModuleProtocol::Subc,
9700 overlap: Default::default(),
9701 })
9702 .unwrap();
9703
9704 let deadline = Instant::now() + Duration::from_secs(5);
9705 loop {
9706 let tail = module.stderr_tail(None, None);
9707 if tail
9708 .entries
9709 .iter()
9710 .any(|entry| matches!(entry, TailEntry::ProcessStart))
9711 && tail.entries.iter().any(|entry| {
9712 matches!(
9713 entry,
9714 TailEntry::Line {
9715 truncated: true,
9716 ..
9717 }
9718 )
9719 })
9720 {
9721 break;
9722 }
9723 assert!(
9724 Instant::now() < deadline,
9725 "module did not produce a truncated line and restart boundary: {tail:?}"
9726 );
9727 sleep(Duration::from_millis(10)).await;
9728 }
9729
9730 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9731 let request = ClientControlRequest::SupervisorStderrTail {
9732 module_id: "stderr-tail-wire".to_string(),
9733 max_lines: None,
9734 max_bytes: None,
9735 };
9736 let frame = Frame::build(
9737 FrameType::Request,
9738 control_flags(),
9739 0,
9740 0,
9741 1,
9742 serde_json::to_vec(&request).unwrap(),
9743 )
9744 .unwrap();
9745 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9746 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9747 let ClientControlResponse::SupervisorStderrTail { tail, .. } =
9748 serde_json::from_slice(&responses[0].body).unwrap()
9749 else {
9750 panic!("expected supervisor.stderr_tail response");
9751 };
9752
9753 assert!(
9754 tail.entries
9755 .iter()
9756 .any(|entry| matches!(entry, StderrTailEntry::ProcessStart)),
9757 "the control response lost the restart boundary"
9758 );
9759 let Some(StderrTailEntry::Line {
9760 text,
9761 truncated,
9762 at_ms,
9763 }) = tail.entries.iter().find(|entry| {
9764 matches!(
9765 entry,
9766 StderrTailEntry::Line {
9767 truncated: true,
9768 ..
9769 }
9770 )
9771 })
9772 else {
9773 panic!("the control response lost the truncated line");
9774 };
9775 assert_eq!(text, &source_line[..DEFAULT_MAX_LINE_BYTES]);
9776 assert!(*truncated);
9777 assert!(
9778 at_ms.is_some(),
9779 "the control response lost the line's capture time"
9780 );
9781 }
9782
9783 #[tokio::test(flavor = "current_thread")]
9788 async fn supervisor_terminals_reads_the_journal_off_the_runtime_worker() {
9789 let dir = ScratchDir::new("terminals-off-worker");
9790 let journal_path = dir.join("terminals.jsonl");
9791 let registry = Arc::new(Registry::default());
9792 let supervisor_handle = SupervisorHandle::new();
9793 let supervisor =
9794 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9795 .with_handle(supervisor_handle.clone())
9796 .with_terminal_journal(journal_path.clone(), "off-worker-daemon".to_string());
9797 let module = supervisor
9798 .spawn(ModuleSpec {
9799 carry_containment: false,
9800 module_id: "terminal-off-worker".to_string(),
9801 program: fake_aft_stub_path(),
9802 args: Vec::new(),
9803 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9804 reserved: false,
9805 reserved_prefixes: Vec::new(),
9806 protocol: ModuleProtocol::Subc,
9807 overlap: Default::default(),
9808 })
9809 .unwrap();
9810 let deadline = Instant::now() + Duration::from_secs(5);
9811 while module.terminal_history().entries.len() != 2 {
9812 assert!(Instant::now() < deadline, "module did not record two exits");
9813 sleep(Duration::from_millis(10)).await;
9814 }
9815
9816 let (started, release) = crate::terminal_journal::read_pause::install(&journal_path);
9817 let handler =
9818 Arc::new(ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle));
9819 let frame = Frame::build(
9820 FrameType::Request,
9821 control_flags(),
9822 0,
9823 0,
9824 1,
9825 serde_json::to_vec(&ClientControlRequest::SupervisorTerminals {
9826 module_id: "terminal-off-worker".to_string(),
9827 })
9828 .unwrap(),
9829 )
9830 .unwrap();
9831 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9832 let spawned_at = std::time::Instant::now();
9833 let read = tokio::spawn({
9834 let handler = Arc::clone(&handler);
9835 async move { handler.handle_control_frame(&ctx, frame).await }
9836 });
9837 tokio::task::spawn_blocking(move || started.recv_timeout(Duration::from_secs(5)))
9840 .await
9841 .unwrap()
9842 .expect("the history read reached its pause");
9843 let elapsed = spawned_at.elapsed();
9844 assert!(
9845 elapsed < Duration::from_secs(2) && !read.is_finished(),
9846 "this task could not run while the history read was paused \
9847 (resumed after {elapsed:?}, read finished: {})",
9848 read.is_finished()
9849 );
9850
9851 drop(release);
9852 let responses = read.await.unwrap().unwrap();
9853 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9854 let ClientControlResponse::SupervisorTerminals { terminals, .. } = response else {
9855 panic!("expected supervisor.terminals response");
9856 };
9857 assert_eq!(terminals.entries.len(), 2);
9858 assert_eq!(terminals.journal_skipped_lines, 0);
9859 assert_eq!(terminals.journal_read_errors, 0);
9860 }
9861
9862 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
9863 async fn supervisor_terminals_golden_is_generated_through_the_real_handler() {
9864 let registry = Arc::new(Registry::default());
9865 let supervisor_handle = SupervisorHandle::new();
9866 let supervisor =
9867 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(1, Duration::ZERO))
9868 .with_handle(supervisor_handle.clone());
9869 let module = supervisor
9870 .spawn(ModuleSpec {
9871 carry_containment: false,
9872 module_id: "terminal-golden".to_string(),
9873 program: fake_aft_stub_path(),
9874 args: Vec::new(),
9875 env: vec![("FAKE_AFT_EXIT_CODE".to_string(), "23".to_string())],
9876 reserved: false,
9877 reserved_prefixes: Vec::new(),
9878 protocol: ModuleProtocol::Subc,
9879 overlap: Default::default(),
9880 })
9881 .unwrap();
9882
9883 let deadline = Instant::now() + Duration::from_secs(5);
9884 while module.terminal_history().entries.len() != 2 {
9885 assert!(
9886 Instant::now() < deadline,
9887 "module did not retain two terminal exits: {:?}",
9888 module.terminal_history()
9889 );
9890 sleep(Duration::from_millis(10)).await;
9891 }
9892
9893 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
9894 let request = ClientControlRequest::SupervisorTerminals {
9895 module_id: "terminal-golden".to_string(),
9896 };
9897 let frame = Frame::build(
9898 FrameType::Request,
9899 control_flags(),
9900 0,
9901 0,
9902 1,
9903 serde_json::to_vec(&request).unwrap(),
9904 )
9905 .unwrap();
9906 let (ctx, _egress) = route_ctx(ConnectionId::new(1));
9907 let responses = handler.handle_control_frame(&ctx, frame).await.unwrap();
9908 let response: ClientControlResponse = serde_json::from_slice(&responses[0].body).unwrap();
9909 let ClientControlResponse::SupervisorTerminals { terminals, .. } = &response else {
9910 panic!("expected supervisor.terminals response");
9911 };
9912 assert_eq!(terminals.entries.len(), 2);
9913 assert_eq!(terminals.dropped, 0);
9914
9915 let mut rendered = serde_json::to_value(response).unwrap();
9916 rendered["daemon_started_at_ms"] = json!(1_700_000_000_000u64);
9919 for (index, entry) in rendered["entries"]
9920 .as_array_mut()
9921 .expect("terminal response entries array")
9922 .iter_mut()
9923 .enumerate()
9924 {
9925 entry["at_ms"] = json!(1_700_000_000_001u64 + index as u64);
9926 }
9927
9928 let golden_path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
9929 .join("../subc-control/tests/golden/client_control_response_supervisor_terminals.json");
9930 let serialized = serde_json::to_string_pretty(&rendered).unwrap() + "\n";
9931 if std::env::var_os("UPDATE_GOLDEN").is_some() {
9932 std::fs::write(&golden_path, &serialized).unwrap();
9933 }
9934 let expected: Value =
9935 serde_json::from_str(&std::fs::read_to_string(&golden_path).unwrap()).unwrap();
9936 assert_eq!(rendered, expected);
9937 }
9938
9939 #[test]
9940 fn hello_registers_manifest_and_returns_ack() {
9941 let registry = Arc::new(Registry::default());
9942 let handler = ControlHandler::new(Arc::clone(®istry));
9943 let conn = ConnectionId::new(1);
9944
9945 let responses = handler
9946 .handle_control(conn, hello_frame("aft", PROTOCOL_VERSION, 7))
9947 .unwrap();
9948
9949 assert_eq!(responses.len(), 1);
9950 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
9951 assert_eq!(responses[0].header.channel, 0);
9952 assert_eq!(responses[0].header.corr, 7);
9953 let ack = parse_ack(&responses[0]);
9954 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
9955 assert!(ack
9956 .subc_capabilities
9957 .contains(&CAP_MANIFEST_REGISTRATION.to_string()));
9958 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_LIST.to_string()));
9959 assert!(ack.subc_ops.contains(&ops::SUPERVISOR_RESTART.to_string()));
9960 assert!(ack
9961 .subc_ops
9962 .contains(&ops::SUPERVISOR_SET_ENABLED.to_string()));
9963 assert!(ack
9964 .subc_ops
9965 .contains(&MODULE_TO_SUBC_OP_CATALOG_UPDATE.to_string()));
9966
9967 let registration = registry.get_module("aft").unwrap().unwrap();
9968 assert_eq!(registration.negotiated_ver, PROTOCOL_VERSION);
9969 assert_eq!(registration.state, ChannelState::Active);
9970 assert_eq!(registration.connection_id, conn);
9971 assert_eq!(registration.control_ops, module_baseline_control_ops());
9972 assert!(registration.manifest.display_name.is_none());
9973 }
9974
9975 #[test]
9976 fn hello_display_name_refuses_invalid_manifest_by_field() {
9977 let registry = Arc::new(Registry::default());
9978 let handler = ControlHandler::new(Arc::clone(®istry));
9979 for (index, invalid) in [
9980 json!(""),
9981 json!("a".repeat(41)),
9982 json!(" Routing"),
9983 json!("a\nb"),
9984 json!("a\u{202e}b"),
9985 json!(7),
9986 ]
9987 .into_iter()
9988 .enumerate()
9989 {
9990 let module_id = format!("bad-display-name-{index}");
9991 let mut body = serde_json::to_value(ModuleHelloBody {
9992 manifest: manifest(&module_id, PROTOCOL_VERSION),
9993 protocol_ver: PROTOCOL_VERSION,
9994 control_ops: None,
9995 launch_nonce: None,
9996 })
9997 .unwrap();
9998 body["manifest"]["display_name"] = invalid;
9999 let hello = Frame::build(
10000 FrameType::Hello,
10001 control_flags(),
10002 0,
10003 0,
10004 7,
10005 serde_json::to_vec(&body).unwrap(),
10006 )
10007 .unwrap();
10008 let response = handler
10009 .handle_control(ConnectionId::new(100 + index as u64), hello)
10010 .unwrap();
10011 assert_eq!(response[0].header.ty, FrameType::Error);
10012 let error = parse_error(&response[0]);
10013 assert_eq!(error["code"], "invalid_manifest");
10014 assert!(error["message"].as_str().unwrap().contains("display_name"));
10015 assert!(registry.get_module(&module_id).unwrap().is_none());
10016 }
10017 }
10018
10019 #[test]
10020 fn capability_grammar_refusals_name_the_field_and_leave_no_catalog_entry() {
10021 let invalid_identifiers = [
10022 ("case_change", "credentials-Provider/v1"),
10023 ("leading_zero", "credentials-provider/v01"),
10024 ("trailing_hyphen", "credentials-provider-/v1"),
10025 ("consecutive_hyphens", "credentials--provider/v1"),
10026 ("uppercase", "Credentials-provider/v1"),
10027 ("missing_v", "credentials-provider/1"),
10028 ("whitespace", "credentials provider/v1"),
10029 ("zero_version", "credentials-provider/v0"),
10030 ("out_of_range_version", "credentials-provider/v4294967296"),
10031 (
10032 "overlength_name",
10033 "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/v1",
10034 ),
10035 ];
10036 let mut cases = invalid_identifiers
10037 .into_iter()
10038 .map(|(name, identifier)| {
10039 (
10040 format!("identifier_{name}"),
10041 "capabilities.provides[0]".to_string(),
10042 identifier.to_string(),
10043 json!({ "provides": [identifier] }),
10044 None,
10045 )
10046 })
10047 .collect::<Vec<_>>();
10048 cases.extend([
10049 (
10050 "unknown_need".to_string(),
10051 "capabilities.requires[0].need".to_string(),
10052 "deferred".to_string(),
10053 json!({ "requires": [{ "capability": "credentials-provider/v1", "need": "deferred" }] }),
10054 None,
10055 ),
10056 (
10057 "duplicate_provides".to_string(),
10058 "capabilities.provides[1]".to_string(),
10059 "credentials-provider/v1".to_string(),
10060 json!({ "provides": ["credentials-provider/v1", "credentials-provider/v1"] }),
10061 None,
10062 ),
10063 (
10064 "duplicate_must_never_reach".to_string(),
10065 "capabilities.must_never_reach[1]".to_string(),
10066 "credentials-provider/v1".to_string(),
10067 json!({ "must_never_reach": ["credentials-provider/v1", "credentials-provider/v1"] }),
10068 None,
10069 ),
10070 (
10071 "duplicate_requires_same_need".to_string(),
10072 "capabilities.requires[1]".to_string(),
10073 "credentials-provider/v1".to_string(),
10074 json!({ "requires": [
10075 { "capability": "credentials-provider/v1", "need": "required" },
10076 { "capability": "credentials-provider/v1", "need": "required" }
10077 ] }),
10078 None,
10079 ),
10080 (
10081 "duplicate_requires_conflicting_need".to_string(),
10082 "capabilities.requires[1]".to_string(),
10083 "credentials-provider/v1".to_string(),
10084 json!({ "requires": [
10085 { "capability": "credentials-provider/v1", "need": "required" },
10086 { "capability": "credentials-provider/v1", "need": "optional" }
10087 ] }),
10088 None,
10089 ),
10090 (
10091 "capabilities_root_pointer".to_string(),
10092 "runtime_computed[0]".to_string(),
10093 "/capabilities".to_string(),
10094 json!({}),
10095 Some(json!(["/capabilities"])),
10096 ),
10097 (
10098 "capabilities_descendant_pointer".to_string(),
10099 "runtime_computed[0]".to_string(),
10100 "/capabilities/provides".to_string(),
10101 json!({}),
10102 Some(json!(["/capabilities/provides"])),
10103 ),
10104 (
10105 "malformed_pointer_without_leading_slash".to_string(),
10106 "runtime_computed[0]".to_string(),
10107 "capabilities".to_string(),
10108 json!({}),
10109 Some(json!(["capabilities"])),
10110 ),
10111 (
10112 "malformed_pointer_escape".to_string(),
10113 "runtime_computed[0]".to_string(),
10114 "/roles/~2/tools".to_string(),
10115 json!({}),
10116 Some(json!(["/roles/~2/tools"])),
10117 ),
10118 (
10119 "unknown_capabilities_field".to_string(),
10120 "capabilities.future".to_string(),
10121 "<array>".to_string(),
10122 json!({ "future": [] }),
10123 None,
10124 ),
10125 ]);
10126
10127 for (index, (name, field, value, capabilities, runtime_computed)) in
10128 cases.into_iter().enumerate()
10129 {
10130 let registry = Arc::new(Registry::default());
10131 let handler = ControlHandler::new(Arc::clone(®istry));
10132 let response = handler
10133 .handle_control(
10134 ConnectionId::new((index + 1) as u64),
10135 capability_grammar_hello_frame(
10136 capabilities,
10137 runtime_computed,
10138 index as u64 + 1,
10139 ),
10140 )
10141 .expect("invalid HELLO returns a refusal");
10142
10143 assert_eq!(response.len(), 1, "{name} must emit one refusal");
10144 let error = parse_error(&response[0]);
10145 assert_eq!(error["code"], "invalid_capability_grammar", "{name}");
10146 let message = error["message"]
10147 .as_str()
10148 .expect("error message is a string");
10149 assert!(
10150 message.contains(&field),
10151 "{name}: field missing from {message}"
10152 );
10153 assert!(
10154 message.contains(&value),
10155 "{name}: value missing from {message}"
10156 );
10157 assert_eq!(
10158 registry
10159 .active_registration_count()
10160 .expect("registry reads"),
10161 0,
10162 "{name}: refused HELLO must not create a catalog entry"
10163 );
10164 }
10165 }
10166
10167 #[test]
10168 fn legal_runtime_pointer_and_capabilities_are_mirrored_in_catalog_list() {
10169 let registry = Arc::new(Registry::default());
10170 let handler = ControlHandler::new(Arc::clone(®istry));
10171 let capabilities = json!({
10172 "provides": ["credentials-provider/v1"],
10173 "requires": [{ "capability": "context-transform/v1", "need": "optional" }],
10174 "must_never_reach": ["federation-transport/v1"]
10175 });
10176 let response = handler
10177 .handle_control(
10178 ConnectionId::new(99),
10179 capability_grammar_hello_frame(
10180 capabilities.clone(),
10181 Some(json!(["/roles/0/tools"])),
10182 99,
10183 ),
10184 )
10185 .expect("valid HELLO registers");
10186 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10187
10188 let request = Frame::build(
10189 FrameType::Request,
10190 control_flags(),
10191 0,
10192 0,
10193 100,
10194 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10195 .expect("catalog request serializes"),
10196 )
10197 .expect("catalog request frame builds");
10198 let response = handler
10199 .handle_catalog_list(request, None)
10200 .expect("catalog list succeeds");
10201 let ClientControlResponse::CatalogList { modules, .. } =
10202 serde_json::from_slice(&response[0].body).expect("catalog response decodes")
10203 else {
10204 panic!("catalog request must return catalog.list");
10205 };
10206 assert_eq!(modules.len(), 1);
10207 assert_eq!(
10208 serde_json::to_value(&modules[0].capabilities).expect("catalog capabilities serialize"),
10209 capabilities
10210 );
10211 }
10212
10213 #[test]
10214 fn catalog_list_mirrors_management_operation_description() {
10215 let registry = Arc::new(Registry::default());
10216 let handler = ControlHandler::new(Arc::clone(®istry));
10217 let description = "List managed records and return their identifiers and metadata.";
10218 let mut manifest = manifest("described-management", PROTOCOL_VERSION);
10219 manifest.provides = vec![ProviderRole::ManagementSurface {
10220 operations: vec![ManagementOperation {
10221 name: "records.list".to_string(),
10222 kind: ManagementOperationKind::Query,
10223 description: Some(description.to_string()),
10224 }],
10225 config_schema: json!({"type": "object"}),
10226 observability: vec![ObservabilitySurface {
10227 name: "records.stats".to_string(),
10228 kind: ObservabilityKind::Snapshot,
10229 }],
10230 identity_scope: vec![IdentityScope::Project],
10231 concurrency: Concurrency::ModuleManaged,
10232 }];
10233 registry
10234 .register_with_control_ops(
10235 manifest,
10236 PROTOCOL_VERSION,
10237 ConnectionId::new(99),
10238 Vec::new(),
10239 )
10240 .expect("described management manifest registers");
10241
10242 let request = Frame::build(
10243 FrameType::Request,
10244 control_flags(),
10245 0,
10246 0,
10247 100,
10248 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
10249 .expect("catalog request serializes"),
10250 )
10251 .expect("catalog request frame builds");
10252 let response = handler
10253 .handle_catalog_list(request, None)
10254 .expect("catalog list succeeds");
10255 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
10256 assert_eq!(
10257 body["modules"][0]["roles"][0]["operations"][0]["description"], description,
10258 "catalog.list must preserve the declared operation description verbatim"
10259 );
10260 }
10261
10262 #[test]
10263 fn catalog_list_mirrors_registered_operation_delivery() {
10264 let registry = Arc::new(Registry::default());
10265 let handler = ControlHandler::new(Arc::clone(®istry));
10266 let delivery =
10267 json!({"status": "watch", "future": "stream-v2", "reset": "watch", "missing": "watch"});
10268 let mut body = serde_json::to_value(ModuleHelloBody {
10269 manifest: manifest("watch-management", PROTOCOL_VERSION),
10270 protocol_ver: PROTOCOL_VERSION,
10271 control_ops: None,
10272 launch_nonce: None,
10273 })
10274 .unwrap();
10275 body["manifest"]["provides"] = json!([{
10276 "role": "management_surface",
10277 "operations": [
10278 {"name": "status", "kind": "query"},
10279 {"name": "future", "kind": "query"},
10280 {"name": "reset", "kind": "mutate"}
10281 ],
10282 "config_schema": {}, "observability": [], "identity_scope": []
10283 }]);
10284 body["manifest"]["operation_delivery"] = delivery.clone();
10285 let hello = Frame::build(
10286 FrameType::Hello,
10287 control_flags(),
10288 0,
10289 0,
10290 99,
10291 serde_json::to_vec(&body).unwrap(),
10292 )
10293 .unwrap();
10294 let response = handler
10295 .handle_control(ConnectionId::new(99), hello)
10296 .unwrap();
10297 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10298
10299 let (_, registrations) = registry.list_modules().unwrap();
10300 let registered = ®istrations[0].manifest;
10301 assert!(registered.is_watch_operation("status"));
10302 assert!(!registered.is_watch_operation("future"));
10303 assert!(!registered.is_watch_operation("reset"));
10304 assert!(!registered.is_watch_operation("missing"));
10305
10306 let request = Frame::build(
10307 FrameType::Request,
10308 control_flags(),
10309 0,
10310 0,
10311 100,
10312 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10313 )
10314 .unwrap();
10315 let response = handler.handle_catalog_list(request, None).unwrap();
10316 let body: Value = serde_json::from_slice(&response[0].body).unwrap();
10317 assert_eq!(body["modules"][0]["operation_delivery"], delivery);
10318 let ClientControlResponse::CatalogList { modules, .. } =
10319 serde_json::from_slice(&response[0].body).unwrap()
10320 else {
10321 panic!("expected catalog.list");
10322 };
10323 assert_eq!(modules[0].operation_delivery, registered.operation_delivery);
10324 }
10325
10326 #[test]
10327 fn catalog_list_mirrors_rich_ui_through_hello_and_catalog_update() {
10328 let registry = Arc::new(Registry::default());
10329 let supervisor = SupervisorHandle::new();
10330 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
10331 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10332 let descriptor = json!({
10333 "schema": "ckios.richui.manifest/1",
10334 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10335 "bytes": 4096,
10336 "chunks": 4
10337 });
10338 let connection = ConnectionId::new(201);
10339 let hello = handler
10340 .handle_control(
10341 connection,
10342 hello_frame_with_rich_ui(
10343 "vault",
10344 PROTOCOL_VERSION,
10345 201,
10346 Some("the-real-nonce"),
10347 descriptor.clone(),
10348 ),
10349 )
10350 .unwrap();
10351 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10352
10353 let request =
10354 Frame::build(FrameType::Request, control_flags(), 0, 0, 202, Vec::new()).unwrap();
10355 let provides = registry
10356 .get_module_by_connection(connection)
10357 .unwrap()
10358 .unwrap()
10359 .manifest
10360 .provides;
10361 handler
10362 .handle_catalog_update(connection, request, provides, None, Some(false))
10363 .unwrap();
10364
10365 let request = Frame::build(
10366 FrameType::Request,
10367 control_flags(),
10368 0,
10369 0,
10370 203,
10371 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10372 )
10373 .unwrap();
10374 let response = handler.handle_catalog_list(request, None).unwrap();
10375 let ClientControlResponse::CatalogList { modules, .. } =
10376 serde_json::from_slice(&response[0].body).unwrap()
10377 else {
10378 panic!("expected catalog.list");
10379 };
10380 assert_eq!(
10381 serde_json::to_value(&modules[0]).unwrap()["rich_ui"],
10382 descriptor
10383 );
10384 assert!(modules[0].nonce_verified);
10385 }
10386
10387 fn hello_with_role_declarations(declarations: Value) -> Frame {
10388 let mut body = serde_json::to_value(ModuleHelloBody {
10389 manifest: manifest("role-test", PROTOCOL_VERSION),
10390 protocol_ver: PROTOCOL_VERSION,
10391 control_ops: None,
10392 launch_nonce: None,
10393 })
10394 .unwrap();
10395 body["manifest"]["role_declarations"] = declarations;
10396 Frame::build(
10397 FrameType::Hello,
10398 control_flags(),
10399 0,
10400 0,
10401 301,
10402 serde_json::to_vec(&body).unwrap(),
10403 )
10404 .unwrap()
10405 }
10406
10407 fn role_declarations_catalog(handler: &ControlHandler) -> (Value, CatalogEntry) {
10408 let request = Frame::build(
10409 FrameType::Request,
10410 control_flags(),
10411 0,
10412 0,
10413 302,
10414 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10415 )
10416 .unwrap();
10417 let response = handler.handle_catalog_list(request, None).unwrap();
10418 let raw: Value = serde_json::from_slice(&response[0].body).unwrap();
10419 let ClientControlResponse::CatalogList { mut modules, .. } =
10420 serde_json::from_slice(&response[0].body).unwrap()
10421 else {
10422 panic!("expected catalog.list");
10423 };
10424 (raw["modules"][0].clone(), modules.remove(0))
10425 }
10426
10427 #[test]
10428 fn catalog_list_mirrors_role_declarations_through_hello_and_catalog_update() {
10429 let registry = Arc::new(Registry::default());
10430 let handler = ControlHandler::new(Arc::clone(®istry));
10431 let declarations = json!({
10432 "routing/v1": {"consumers": [{"name": "chat", "default_demand": 2}]},
10433 "future/v2": {"unknown": [null, true, {"nested": "λ"}]}
10434 });
10435 let connection = ConnectionId::new(301);
10436 let response = handler
10437 .handle_control(
10438 connection,
10439 hello_with_role_declarations(declarations.clone()),
10440 )
10441 .unwrap();
10442 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10443 let registered = registry
10444 .get_module_by_connection(connection)
10445 .unwrap()
10446 .unwrap()
10447 .manifest;
10448 assert_eq!(
10452 serde_json::to_value(®istered.role_declarations).unwrap(),
10453 declarations
10454 );
10455 let (raw, entry) = role_declarations_catalog(&handler);
10456 assert_eq!(raw["role_declarations"], declarations);
10457 assert_eq!(entry.role_declarations, registered.role_declarations);
10458
10459 let request =
10460 Frame::build(FrameType::Request, control_flags(), 0, 0, 303, Vec::new()).unwrap();
10461 handler
10462 .handle_catalog_update(connection, request, registered.provides, None, Some(false))
10463 .unwrap();
10464 let (raw, entry) = role_declarations_catalog(&handler);
10465 assert_eq!(raw["role_declarations"], declarations);
10466 assert_eq!(entry.role_declarations, registered.role_declarations);
10467 }
10468
10469 fn assert_hello_drops_role_declarations_with_warning(declarations: Value, reason: &str) {
10470 let registry = Arc::new(Registry::default());
10471 let handler = ControlHandler::new(Arc::clone(®istry));
10472 let capture = EventCapture::default();
10473 let _subscriber =
10474 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10475 let response = handler
10476 .handle_control(
10477 ConnectionId::new(301),
10478 hello_with_role_declarations(declarations),
10479 )
10480 .unwrap();
10481 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10482 assert!(registry
10483 .get_module_by_connection(ConnectionId::new(301))
10484 .unwrap()
10485 .unwrap()
10486 .manifest
10487 .role_declarations
10488 .is_empty());
10489 let (raw, entry) = role_declarations_catalog(&handler);
10490 assert!(raw.get("role_declarations").is_none());
10491 assert!(entry.role_declarations.is_empty());
10492 let warnings: Vec<_> = capture
10493 .events()
10494 .into_iter()
10495 .filter(|event| {
10496 event.fields.get("message").map(String::as_str)
10497 == Some("dropping whole role_declarations map from HELLO")
10498 })
10499 .collect();
10500 assert_eq!(warnings.len(), 1);
10501 assert_eq!(warnings[0].level, tracing::Level::WARN);
10502 assert_eq!(warnings[0].fields["module_id"], "\"role-test\"");
10503 assert_eq!(warnings[0].fields["reason"], reason);
10504 }
10505
10506 #[test]
10507 fn hello_non_object_role_declarations_drop_whole_map_and_warn() {
10508 for value in [
10509 json!(null),
10510 json!(false),
10511 json!(0),
10512 json!("opaque"),
10513 json!([]),
10514 ] {
10515 assert_hello_drops_role_declarations_with_warning(
10516 json!({"routing/v1": {}, "broken/v1": value}),
10517 "role_declarations values must be objects",
10518 );
10519 }
10520 assert_hello_drops_role_declarations_with_warning(
10521 json!([]),
10522 "role_declarations must be an object",
10523 );
10524 }
10525
10526 #[test]
10527 fn hello_bad_role_declaration_key_drops_whole_map_and_warns() {
10528 assert_hello_drops_role_declarations_with_warning(
10529 json!({"routing/v1": {}, "bad key": {}}),
10530 "role_declarations key must use capability-name grammar",
10531 );
10532 }
10533
10534 #[test]
10535 fn hello_oversized_role_declarations_drop_whole_map_and_warn() {
10536 assert_hello_drops_role_declarations_with_warning(
10537 json!({"routing/v1": {}, "overflow/v1": {"padding": "x".repeat(16_384)}}),
10538 "role_declarations exceeds 16384 compact JSON bytes",
10539 );
10540 }
10541
10542 fn hello_with_flow_operations(capabilities: Value, flow_operations: Option<Value>) -> Frame {
10543 let mut body = serde_json::to_value(ModuleHelloBody {
10544 manifest: manifest("flow-ops", PROTOCOL_VERSION),
10545 protocol_ver: PROTOCOL_VERSION,
10546 control_ops: None,
10547 launch_nonce: None,
10548 })
10549 .unwrap();
10550 body["manifest"]["provides"] = json!([{
10551 "role": "management_surface",
10552 "operations": [
10553 {"name": "diagnostics.findings", "kind": "query"},
10554 {"name": "reset", "kind": "mutate"}
10555 ],
10556 "config_schema": {}, "observability": [], "identity_scope": []
10557 }]);
10558 body["manifest"]["capabilities"] = json!({ "provides": capabilities });
10559 if let Some(flow_operations) = flow_operations {
10560 body["manifest"]["flow_operations"] = flow_operations;
10561 }
10562 Frame::build(
10563 FrameType::Hello,
10564 control_flags(),
10565 0,
10566 0,
10567 401,
10568 serde_json::to_vec(&body).unwrap(),
10569 )
10570 .unwrap()
10571 }
10572
10573 #[test]
10574 fn catalog_list_mirrors_flow_operations_unchanged() {
10575 for (capability, flow_operations) in [
10578 ("flow-scopes/v1", json!(["reset", "diagnostics.findings"])),
10579 ("call-modes/v1", json!(["diagnostics.findings"])),
10580 ("flow-scopes/v1", json!([])),
10581 ] {
10582 let registry = Arc::new(Registry::default());
10583 let handler = ControlHandler::new(Arc::clone(®istry));
10584 let connection = ConnectionId::new(401);
10585 let response = handler
10586 .handle_control(
10587 connection,
10588 hello_with_flow_operations(json!([capability]), Some(flow_operations.clone())),
10589 )
10590 .unwrap();
10591 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10592 let registered = registry
10593 .get_module_by_connection(connection)
10594 .unwrap()
10595 .unwrap()
10596 .manifest;
10597 assert_eq!(
10598 serde_json::to_value(®istered.flow_operations).unwrap(),
10599 flow_operations
10600 );
10601 let (raw, entry) = role_declarations_catalog(&handler);
10602 assert_eq!(raw["flow_operations"], flow_operations);
10603 assert_eq!(entry.flow_operations, registered.flow_operations);
10604 }
10605
10606 let registry = Arc::new(Registry::default());
10607 let handler = ControlHandler::new(Arc::clone(®istry));
10608 let response = handler
10609 .handle_control(
10610 ConnectionId::new(401),
10611 hello_with_flow_operations(json!(["flow-scopes/v1"]), None),
10612 )
10613 .unwrap();
10614 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10615 let (raw, entry) = role_declarations_catalog(&handler);
10616 assert!(raw.get("flow_operations").is_none());
10617 assert_eq!(entry.flow_operations, None);
10618 }
10619
10620 fn assert_hello_drops_flow_operations_with_warning(
10621 capabilities: Value,
10622 flow_operations: Value,
10623 reason: &str,
10624 ) {
10625 let registry = Arc::new(Registry::default());
10626 let handler = ControlHandler::new(Arc::clone(®istry));
10627 let capture = EventCapture::default();
10628 let _subscriber =
10629 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
10630 let response = handler
10631 .handle_control(
10632 ConnectionId::new(401),
10633 hello_with_flow_operations(capabilities, Some(flow_operations)),
10634 )
10635 .unwrap();
10636 assert_eq!(response[0].header.ty, FrameType::HelloAck);
10637 let registered = registry
10638 .get_module_by_connection(ConnectionId::new(401))
10639 .unwrap()
10640 .expect("the module still registers")
10641 .manifest;
10642 assert_eq!(registered.module_id, "flow-ops");
10643 assert_eq!(registered.flow_operations, None);
10644 let (raw, entry) = role_declarations_catalog(&handler);
10645 assert_eq!(raw["module_id"], "flow-ops");
10646 assert!(raw.get("flow_operations").is_none());
10647 assert_eq!(entry.flow_operations, None);
10648 let warnings: Vec<_> = capture
10649 .events()
10650 .into_iter()
10651 .filter(|event| {
10652 event.fields.get("message").map(String::as_str)
10653 == Some("dropping flow_operations from HELLO")
10654 })
10655 .collect();
10656 assert_eq!(warnings.len(), 1);
10657 assert_eq!(warnings[0].level, tracing::Level::WARN);
10658 assert_eq!(warnings[0].fields["module_id"], "\"flow-ops\"");
10659 assert_eq!(warnings[0].fields["reason"], reason);
10660 }
10661
10662 #[test]
10663 fn hello_flow_operations_naming_unknown_operation_is_dropped_with_warning() {
10664 assert_hello_drops_flow_operations_with_warning(
10665 json!(["flow-scopes/v1"]),
10666 json!(["diagnostics.findings", "missing"]),
10667 "flow_operations names 'missing', which is not a declared management operation",
10668 );
10669 }
10670
10671 #[test]
10672 fn hello_flow_operations_without_flow_capability_is_dropped_with_warning() {
10673 assert_hello_drops_flow_operations_with_warning(
10674 json!([]),
10675 json!(["diagnostics.findings"]),
10676 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10677 );
10678 assert_hello_drops_flow_operations_with_warning(
10680 json!(["agent-run-scopes/v1"]),
10681 json!([]),
10682 "flow_operations requires 'flow-scopes/v1' or 'call-modes/v1' in capabilities.provides",
10683 );
10684 }
10685
10686 #[test]
10687 fn hello_malformed_flow_operations_is_dropped_with_warning() {
10688 assert_hello_drops_flow_operations_with_warning(
10689 json!(["flow-scopes/v1"]),
10690 json!(["reset", "reset"]),
10691 "flow_operations names 'reset' more than once",
10692 );
10693 assert_hello_drops_flow_operations_with_warning(
10694 json!(["flow-scopes/v1"]),
10695 json!(vec!["reset"; 257]),
10696 "flow_operations lists 257 names, more than 256",
10697 );
10698 assert_hello_drops_flow_operations_with_warning(
10699 json!(["flow-scopes/v1"]),
10700 json!("reset"),
10701 "flow_operations must be an array of operation names",
10702 );
10703 }
10704
10705 #[test]
10706 fn rich_ui_descriptor_malformed_fields_are_refused_by_name() {
10707 let registry = Arc::new(Registry::default());
10708 let handler = ControlHandler::new(registry.clone());
10709 let descriptor = json!({
10710 "schema": "ckios.richui.manifest/1",
10711 "manifest_digest": format!("sha256:{}", "a".repeat(64)),
10712 "bytes": 128,
10713 "chunks": 2
10714 });
10715 let malformed = [
10716 ("schema", json!("")),
10717 ("schema", json!("a".repeat(65))),
10718 ("schema", json!("Bad schema")),
10719 (
10720 "manifest_digest",
10721 json!(format!("sha256:{}", "A".repeat(64))),
10722 ),
10723 ("manifest_digest", json!("sha256:abc")),
10724 ("bytes", json!(0)),
10725 ("bytes", json!(16_777_217)),
10726 ("bytes", json!("128")),
10727 ("chunks", json!(0)),
10728 ("chunks", json!(1025)),
10729 ("chunks", json!("2")),
10730 ];
10731 for (index, (field, invalid_value)) in malformed.into_iter().enumerate() {
10732 let mut invalid = descriptor.clone();
10733 invalid[field] = invalid_value;
10734 let response = handler
10735 .handle_control(
10736 ConnectionId::new(210 + index as u64),
10737 hello_frame_with_rich_ui(
10738 &format!("bad-rich-ui-{index}"),
10739 PROTOCOL_VERSION,
10740 210 + index as u64,
10741 None,
10742 invalid,
10743 ),
10744 )
10745 .unwrap();
10746 assert_eq!(response[0].header.ty, FrameType::Error);
10747 let error = parse_error(&response[0]);
10748 assert_eq!(error["code"], "invalid_manifest");
10749 assert!(error["message"]
10750 .as_str()
10751 .unwrap()
10752 .contains(&format!("rich_ui.{field}")));
10753 assert!(registry
10754 .get_module(&format!("bad-rich-ui-{index}"))
10755 .unwrap()
10756 .is_none());
10757 }
10758 }
10759
10760 #[test]
10761 fn catalog_update_refuses_malformed_rich_ui_by_field_name() {
10762 let registry = Arc::new(Registry::default());
10763 let handler = ControlHandler::new(Arc::clone(®istry));
10764 let connection = ConnectionId::new(219);
10765 registry
10766 .register_with_control_ops(
10767 ModuleManifest::builder("invalid-rich-ui", "0.1.0")
10768 .rich_ui(Some(subc_protocol::manifest::RichUiDescriptor::new(
10769 "bad schema",
10770 format!("sha256:{}", "a".repeat(64)),
10771 128,
10772 2,
10773 )))
10774 .build(),
10775 PROTOCOL_VERSION,
10776 connection,
10777 Vec::new(),
10778 )
10779 .unwrap();
10780 let request =
10781 Frame::build(FrameType::Request, control_flags(), 0, 0, 219, Vec::new()).unwrap();
10782 let response = handler
10783 .handle_catalog_update(connection, request, Vec::new(), None, None)
10784 .unwrap();
10785 assert_eq!(response[0].header.ty, FrameType::Error);
10786 let error = parse_error(&response[0]);
10787 assert_eq!(error["code"], "invalid_manifest");
10788 assert!(error["message"]
10789 .as_str()
10790 .unwrap()
10791 .contains("rich_ui.schema"));
10792 }
10793
10794 #[test]
10795 fn swap_candidate_hello_is_nonce_verified_after_promotion() {
10796 let registry = Arc::new(Registry::default());
10797 let supervisor = SupervisorHandle::new();
10798 supervisor.open_swap("provider", "candidate-nonce".to_string());
10799 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
10800 let connection = ConnectionId::new(218);
10801 let hello = handler
10802 .handle_control(
10803 connection,
10804 hello_frame_with_nonce("provider", PROTOCOL_VERSION, 218, Some("candidate-nonce")),
10805 )
10806 .unwrap();
10807 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
10808 assert!(registry.list_modules().unwrap().1.is_empty());
10809
10810 registry.promote_candidate("provider").unwrap().unwrap();
10811 let request = Frame::build(
10812 FrameType::Request,
10813 control_flags(),
10814 0,
10815 0,
10816 219,
10817 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10818 )
10819 .unwrap();
10820 let response = handler.handle_catalog_list(request, None).unwrap();
10821 let ClientControlResponse::CatalogList { modules, .. } =
10822 serde_json::from_slice(&response[0].body).unwrap()
10823 else {
10824 panic!("expected catalog.list");
10825 };
10826 assert_eq!(modules.len(), 1);
10827 assert!(modules[0].nonce_verified);
10828 }
10829
10830 #[test]
10831 fn catalog_list_with_provenance_mirrors_declared_build_bytes() {
10832 for provenance_json in [
10833 r#"{"build_git_sha":"0123456789abcdef0123456789abcdef01234567","build_lock_digest":"abababababababababababababababababababababababababababababababab","wire_crate_version":"0.34.0","store_schema_version":"7","launch_nonce_source":"fd"}"#,
10834 r#"{"build_git_sha_absence_reason":"no_git_dir","build_lock_digest":"abababababababababababababababababababababababababababababababab"}"#,
10835 ] {
10836 let registry = Arc::new(Registry::default());
10837 let handler = ControlHandler::new(registry);
10838 let hello = Frame::build(
10839 FrameType::Hello,
10840 control_flags(),
10841 0,
10842 0,
10843 220,
10844 serde_json::to_vec(&ModuleHelloBody {
10845 manifest: ModuleManifest::builder("built", "0.1.0")
10846 .provenance(Some(serde_json::from_str(provenance_json).unwrap()))
10847 .build(),
10848 protocol_ver: PROTOCOL_VERSION,
10849 control_ops: None,
10850 launch_nonce: None,
10851 })
10852 .unwrap(),
10853 )
10854 .unwrap();
10855 assert_eq!(
10856 handler
10857 .handle_control(ConnectionId::new(220), hello)
10858 .unwrap()[0]
10859 .header
10860 .ty,
10861 FrameType::HelloAck
10862 );
10863 let request = Frame::build(
10864 FrameType::Request,
10865 control_flags(),
10866 0,
10867 0,
10868 221,
10869 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10870 )
10871 .unwrap();
10872 let response = handler.handle_catalog_list(request, None).unwrap();
10873 let ClientControlResponse::CatalogList { modules, .. } =
10874 serde_json::from_slice(&response[0].body).unwrap()
10875 else {
10876 panic!("expected catalog.list");
10877 };
10878 assert_eq!(modules.len(), 1);
10879 assert_eq!(
10880 serde_json::to_vec(&modules[0]).unwrap(),
10881 format!(
10882 r#"{{"module_id":"built","ready":true,"module_version":"0.1.0","provenance":{provenance_json},"roles":[],"control_ops":["route.bind","route.status"]}}"#
10883 )
10884 .as_bytes()
10885 );
10886 }
10887 }
10888
10889 #[test]
10890 fn catalog_list_without_rich_ui_or_nonce_keeps_legacy_entry_bytes() {
10891 let registry = Arc::new(Registry::default());
10892 let handler = ControlHandler::new(Arc::clone(®istry));
10893 let hello = Frame::build(
10894 FrameType::Hello,
10895 control_flags(),
10896 0,
10897 0,
10898 220,
10899 serde_json::to_vec(&ModuleHelloBody {
10900 manifest: ModuleManifest::builder("plain", "0.1.0").build(),
10901 protocol_ver: PROTOCOL_VERSION,
10902 control_ops: None,
10903 launch_nonce: Some("unverified".to_string()),
10904 })
10905 .unwrap(),
10906 )
10907 .unwrap();
10908 assert_eq!(
10909 handler
10910 .handle_control(ConnectionId::new(220), hello)
10911 .unwrap()[0]
10912 .header
10913 .ty,
10914 FrameType::HelloAck
10915 );
10916 let request = Frame::build(
10917 FrameType::Request,
10918 control_flags(),
10919 0,
10920 0,
10921 221,
10922 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap(),
10923 )
10924 .unwrap();
10925 let response = handler.handle_catalog_list(request, None).unwrap();
10926 let wire: serde_json::Value = serde_json::from_slice(&response[0].body).unwrap();
10927 assert!(wire["modules"][0].get("provenance").is_none());
10928 let ClientControlResponse::CatalogList { modules, .. } =
10929 serde_json::from_slice(&response[0].body).unwrap()
10930 else {
10931 panic!("expected catalog.list");
10932 };
10933 assert_eq!(
10934 serde_json::to_vec(&modules[0]).unwrap(),
10935 br#"{"module_id":"plain","ready":true,"module_version":"0.1.0","roles":[],"control_ops":["route.bind","route.status"]}"#
10936 );
10937 assert!(!modules[0].nonce_verified);
10938 assert!(modules[0].provenance.is_none());
10939 }
10940
10941 #[test]
10942 fn reserved_capability_refusal_mutation_proof_leaves_no_catalog_entry() {
10943 let registry = Arc::new(Registry::default());
10944 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
10945 [("vault".to_string(), true), ("squatter".to_string(), true)],
10946 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
10947 );
10948 let mut squatter = manifest("squatter", PROTOCOL_VERSION);
10949 squatter.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
10950 provides: vec!["credentials-provider/v1".to_string()],
10951 requires: Vec::new(),
10952 must_never_reach: Vec::new(),
10953 });
10954 let frame = Frame::build(
10955 FrameType::Hello,
10956 control_flags(),
10957 0,
10958 0,
10959 77,
10960 serde_json::to_vec(&ModuleHelloBody {
10961 manifest: squatter,
10962 protocol_ver: PROTOCOL_VERSION,
10963 control_ops: None,
10964 launch_nonce: None,
10965 })
10966 .expect("HELLO serializes"),
10967 )
10968 .expect("HELLO frame builds");
10969 let response = handler
10970 .handle_control(ConnectionId::new(77), frame)
10971 .expect("reserved claim receives a typed refusal");
10972 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
10973 assert_eq!(
10974 registry
10975 .active_registration_count()
10976 .expect("registry reads"),
10977 0,
10978 "a reserved capability refusal must not leave a catalog entry"
10979 );
10980 }
10981
10982 #[test]
10983 fn stale_relay_settlement_cannot_release_the_half_open_probe() {
10984 for settlement in ["timeout", "inconclusive", "drop"] {
10985 let breakers = RouteBindBreakers::default();
10986 let RouteBindAdmission::Admitted {
10987 guard: mut old,
10988 probe: false,
10989 } = breakers.admit("prov")
10990 else {
10991 panic!("ordinary relay admitted")
10992 };
10993 let RouteBindAdmission::Admitted {
10994 guard: mut opener, ..
10995 } = breakers.admit("prov")
10996 else {
10997 panic!("second relay admitted")
10998 };
10999 assert!(
11000 !opener
11001 .record_timeout(1, Duration::ZERO)
11002 .unwrap()
11003 .reopened_after_probe
11004 );
11005 let RouteBindAdmission::Admitted {
11006 guard: mut probe,
11007 probe: true,
11008 } = breakers.admit("prov")
11009 else {
11010 panic!("one cooldown probe admitted")
11011 };
11012 match settlement {
11013 "timeout" => assert!(
11014 !old.record_timeout(1, Duration::ZERO)
11015 .unwrap()
11016 .reopened_after_probe
11017 ),
11018 "inconclusive" => old.record_inconclusive(),
11019 "drop" => drop(old),
11020 _ => unreachable!(),
11021 }
11022 assert!(
11023 matches!(
11024 breakers.admit("prov"),
11025 RouteBindAdmission::Refused {
11026 probe_in_flight: true,
11027 ..
11028 }
11029 ),
11030 "{settlement} of a pre-open relay cannot release the real probe"
11031 );
11032 assert!(
11033 probe
11034 .record_timeout(1, Duration::ZERO)
11035 .unwrap()
11036 .reopened_after_probe
11037 );
11038 assert!(matches!(
11039 breakers.admit("prov"),
11040 RouteBindAdmission::Admitted { probe: true, .. }
11041 ));
11042 }
11043 let breakers = RouteBindBreakers::default();
11044 let admit = || match breakers.admit("prov") {
11045 RouteBindAdmission::Admitted { guard, .. } => guard,
11046 _ => panic!("relay admitted"),
11047 };
11048 admit().record_timeout(1, Duration::ZERO);
11049 let mut old_probe = admit();
11050 breakers.reset_for_new_module_connection("prov");
11051 admit().record_timeout(1, Duration::ZERO);
11052 let _new_probe = admit();
11053 old_probe.record_inconclusive();
11054 assert!(matches!(
11055 breakers.admit("prov"),
11056 RouteBindAdmission::Refused {
11057 probe_in_flight: true,
11058 ..
11059 }
11060 ));
11061 }
11062
11063 #[tokio::test]
11064 async fn catalog_update_refuses_reserved_capabilities_for_active_and_candidate() {
11065 for candidate in [false, true] {
11066 let registry = Arc::new(Registry::default());
11067 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
11068 [("vault".to_string(), true), ("squatter".to_string(), true)],
11069 BTreeMap::from([("credentials-provider/v1".to_string(), "vault".to_string())]),
11070 );
11071 let conn = ConnectionId::new(77);
11072 let (ctx, mut rx) = route_ctx(conn);
11073 let initial = capability_manifest("squatter", &[], &[]);
11074 if candidate {
11075 registry
11076 .register_candidate_with_control_ops(
11077 initial.clone(),
11078 PROTOCOL_VERSION,
11079 conn,
11080 module_baseline_control_ops(),
11081 )
11082 .unwrap();
11083 handler
11084 .forwarding
11085 .register_candidate_module_connection(
11086 conn,
11087 "squatter".to_string(),
11088 PROTOCOL_VERSION,
11089 manifest_concurrency(&initial),
11090 ctx.egress.clone(),
11091 )
11092 .unwrap();
11093 } else {
11094 hello_via_sink(
11095 &handler,
11096 &ctx,
11097 &mut rx,
11098 hello_frame_with_manifest(initial.clone(), 1),
11099 )
11100 .await;
11101 }
11102 let response = handler
11103 .handle_control_frame(
11104 &ctx,
11105 catalog_update_with_capabilities_frame(
11106 2,
11107 capability_manifest("squatter", &["credentials-provider/v1"], &[])
11108 .capabilities
11109 .unwrap(),
11110 ),
11111 )
11112 .await
11113 .unwrap();
11114 assert_eq!(parse_error(&response[0])["code"], "reserved_capability");
11115 assert_eq!(
11116 registry
11117 .get_module_by_connection(conn)
11118 .unwrap()
11119 .unwrap()
11120 .manifest,
11121 initial
11122 );
11123 }
11124 }
11125
11126 #[test]
11127 fn server_describe_surfaces_required_capability_verdict_fields() {
11128 let registry = Arc::new(Registry::default());
11129 let handler = ControlHandler::new(Arc::clone(®istry)).with_capability_config(
11130 [
11131 ("consumer".to_string(), true),
11132 ("provider".to_string(), false),
11133 ],
11134 BTreeMap::new(),
11135 );
11136 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
11137 consumer.capabilities = Some(subc_protocol::manifest::CapabilityDeclarations {
11138 provides: Vec::new(),
11139 requires: vec![subc_protocol::manifest::CapabilityRequirement {
11140 capability: "credentials-provider/v1".to_string(),
11141 need: subc_protocol::manifest::CapabilityNeed::Required,
11142 }],
11143 must_never_reach: Vec::new(),
11144 });
11145 let hello = Frame::build(
11146 FrameType::Hello,
11147 control_flags(),
11148 0,
11149 0,
11150 78,
11151 serde_json::to_vec(&ModuleHelloBody {
11152 manifest: consumer,
11153 protocol_ver: PROTOCOL_VERSION,
11154 control_ops: None,
11155 launch_nonce: None,
11156 })
11157 .expect("HELLO serializes"),
11158 )
11159 .expect("HELLO frame builds");
11160 handler
11161 .handle_control(ConnectionId::new(78), hello)
11162 .expect("consumer registers");
11163 let describe = Frame::build(
11164 FrameType::Request,
11165 control_flags(),
11166 0,
11167 0,
11168 79,
11169 serde_json::to_vec(&ClientControlRequest::ServerDescribe {})
11170 .expect("request serializes"),
11171 )
11172 .expect("describe frame builds");
11173 let response = handler
11174 .handle_server_describe(describe)
11175 .expect("server.describe succeeds");
11176 let rendered: Value = serde_json::from_slice(&response[0].body).expect("response JSON");
11177 let requirement = &rendered["capability_requirements"][0];
11178 assert_eq!(requirement["consumer"], "consumer");
11179 assert_eq!(requirement["verdict"], "never_provided");
11180 assert_eq!(requirement["episode_seq"], 1);
11181 assert_eq!(requirement["config_satisfiable"], false);
11182 assert_eq!(requirement["runtime_available"], false);
11183 assert!(requirement["detail"]
11184 .as_str()
11185 .expect("detail string")
11186 .contains("credentials-provider/v1"));
11187 }
11188
11189 fn deregister_required_provider(
11195 daemon_shutdown_begun: bool,
11196 ) -> (Vec<CapturedEvent>, Option<CapabilityVerdict>) {
11197 const CAPABILITY: &str = "project-identity/v1";
11198 let registry = Arc::new(Registry::default());
11199 let daemon_shutdown = DaemonShutdownFlag::default();
11200 let handler = ControlHandler::new(Arc::clone(®istry))
11201 .with_capability_config(
11202 [
11203 ("consumer".to_string(), true),
11204 ("provider".to_string(), true),
11205 ],
11206 BTreeMap::new(),
11207 )
11208 .with_daemon_shutdown_flag(daemon_shutdown.clone());
11209 let mut consumer = manifest("consumer", PROTOCOL_VERSION);
11210 consumer.capabilities = Some(CapabilityDeclarations {
11211 provides: Vec::new(),
11212 requires: vec![subc_protocol::manifest::CapabilityRequirement {
11213 capability: CAPABILITY.to_string(),
11214 need: subc_protocol::manifest::CapabilityNeed::Required,
11215 }],
11216 must_never_reach: Vec::new(),
11217 });
11218 let provider = capability_manifest("provider", &[CAPABILITY], &[]);
11219 let provider_conn = ConnectionId::new(182);
11220 for (conn, manifest, corr) in [
11221 (ConnectionId::new(181), consumer, 181),
11222 (provider_conn, provider, 182),
11223 ] {
11224 let hello = handler
11225 .handle_control(conn, hello_frame_with_manifest(manifest, corr))
11226 .expect("module registers");
11227 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11228 }
11229 assert_eq!(
11230 handler.capability_evaluator.verdict("consumer", CAPABILITY),
11231 Some(CapabilityVerdict::Provided),
11232 "the registered provider must satisfy the requirement first"
11233 );
11234 if daemon_shutdown_begun {
11235 daemon_shutdown.set_for_test();
11236 }
11237
11238 let capture = EventCapture::default();
11239 let _subscriber =
11240 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
11241 handler
11242 .cleanup_connection(provider_conn)
11243 .expect("provider deregisters");
11244 let events = capture
11245 .events()
11246 .into_iter()
11247 .filter(|event| {
11248 event.fields.get("message").map(String::as_str) == Some("capability.requirement")
11249 })
11250 .collect();
11251 (
11252 events,
11253 handler.capability_evaluator.verdict("consumer", CAPABILITY),
11254 )
11255 }
11256
11257 #[test]
11258 fn required_provider_leaving_during_daemon_shutdown_logs_info_with_shutdown_cause() {
11259 let (events, verdict) = deregister_required_provider(true);
11260 assert_eq!(
11261 verdict,
11262 Some(CapabilityVerdict::NeverProvided),
11263 "shutdown must not change the verdict route.open refusals read"
11264 );
11265 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11266 let event = &events[0];
11267 assert_eq!(event.level, tracing::Level::INFO, "{event:?}");
11268 assert_eq!(
11269 event.fields.get("cause").map(String::as_str),
11270 Some("\"daemon_shutdown\""),
11271 "{event:?}"
11272 );
11273 assert_eq!(
11274 event.fields.get("verdict").map(String::as_str),
11275 Some("\"never_provided\""),
11276 "{event:?}"
11277 );
11278 }
11279
11280 #[test]
11281 fn required_provider_leaving_while_daemon_runs_still_logs_error() {
11282 let (events, verdict) = deregister_required_provider(false);
11283 assert_eq!(verdict, Some(CapabilityVerdict::NeverProvided));
11284 assert_eq!(events.len(), 1, "one requirement transition: {events:?}");
11285 let event = &events[0];
11286 assert_eq!(event.level, tracing::Level::ERROR, "{event:?}");
11287 assert_eq!(event.fields.get("cause"), None, "{event:?}");
11288 assert_eq!(
11289 event.fields.get("verdict").map(String::as_str),
11290 Some("\"never_provided\""),
11291 "{event:?}"
11292 );
11293 }
11294
11295 #[test]
11296 fn catalog_list_omits_capabilities_for_legacy_manifest() {
11297 let registry = Arc::new(Registry::default());
11298 let handler = ControlHandler::new(Arc::clone(®istry));
11299 let hello = handler
11300 .handle_control(
11301 ConnectionId::new(101),
11302 hello_frame("legacy-capability-manifest", PROTOCOL_VERSION, 101),
11303 )
11304 .expect("legacy HELLO registers");
11305 assert_eq!(hello[0].header.ty, FrameType::HelloAck);
11306
11307 let request = Frame::build(
11308 FrameType::Request,
11309 control_flags(),
11310 0,
11311 0,
11312 102,
11313 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None })
11314 .expect("catalog request serializes"),
11315 )
11316 .expect("catalog request frame builds");
11317 let response = handler
11318 .handle_catalog_list(request, None)
11319 .expect("catalog list succeeds");
11320 let body: Value = serde_json::from_slice(&response[0].body).expect("catalog response JSON");
11321 assert!(
11322 body["modules"][0].get("capabilities").is_none(),
11323 "legacy manifest must retain an absent capabilities field on catalog.list"
11324 );
11325 }
11326
11327 #[test]
11328 fn hello_ack_omits_storage_when_no_storage_config() {
11329 let registry = Arc::new(Registry::default());
11330 let handler = ControlHandler::new(Arc::clone(®istry));
11331 let responses = handler
11332 .handle_control(
11333 ConnectionId::new(1),
11334 hello_frame("aft", PROTOCOL_VERSION, 7),
11335 )
11336 .unwrap();
11337 let ack = parse_ack(&responses[0]);
11338 assert_eq!(ack.storage, None, "no storage config -> no descriptor");
11339 assert_eq!(ack.machine_id, None, "no machine id configured -> no field");
11340 }
11341
11342 #[tokio::test]
11343 async fn hello_ack_and_server_describe_carry_the_configured_machine_id() {
11344 let id = crate::machine_id::MachineId::parse("0123456789abcdef0123456789abcdef").unwrap();
11345 let registry = Arc::new(Registry::default());
11346 let handler = ControlHandler::new(Arc::clone(®istry)).with_machine_id(Some(id.clone()));
11347 let responses = handler
11348 .handle_control(
11349 ConnectionId::new(1),
11350 hello_frame("aft", PROTOCOL_VERSION, 7),
11351 )
11352 .unwrap();
11353 let ack = parse_ack(&responses[0]);
11354 assert_eq!(ack.machine_id.as_deref(), Some(id.as_str()));
11355
11356 let described = handler
11357 .handle_control_frame(
11358 &route_ctx(ConnectionId::new(2)).0,
11359 Frame::build(
11360 FrameType::Request,
11361 control_flags(),
11362 0,
11363 0,
11364 9,
11365 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
11366 )
11367 .unwrap(),
11368 )
11369 .await
11370 .unwrap();
11371 let ClientControlResponse::ServerDescribe { machine_id, .. } =
11372 serde_json::from_slice(&described[0].body).unwrap()
11373 else {
11374 panic!("server.describe answered with another shape");
11375 };
11376 assert_eq!(machine_id.as_deref(), Some(id.as_str()));
11377 }
11378
11379 #[test]
11380 fn hello_ack_delivers_resolved_storage_descriptor_per_module() {
11381 let registry = Arc::new(Registry::default());
11384 let handler = ControlHandler::new(Arc::clone(®istry)).with_storage_config(Some(
11385 crate::daemon_config::StorageConfig::Sqlite {
11386 data_home: std::path::PathBuf::from("/data"),
11387 },
11388 ));
11389
11390 let responses = handler
11391 .handle_control(
11392 ConnectionId::new(1),
11393 hello_frame("alfonso-routing", PROTOCOL_VERSION, 7),
11394 )
11395 .unwrap();
11396 let ack = parse_ack(&responses[0]);
11397 assert_eq!(
11398 ack.storage,
11399 Some(serde_json::json!({
11400 "module_id": "alfonso-routing",
11401 "storage_namespace": "default",
11402 "isolation": { "kind": "module" },
11403 "backend": {
11404 "backend": "sqlite",
11405 "path": "/data/cortexkit/alfonso-routing/store.db"
11406 }
11407 })),
11408 "the delivered descriptor is the module's own sqlite store path"
11409 );
11410 }
11411
11412 #[test]
11413 fn hello_control_ops_none_is_baseline_and_guard_rejects_synthetic_gated_op() {
11414 let registry = Arc::new(Registry::default());
11415 let handler = ControlHandler::new(Arc::clone(®istry));
11416 let conn = ConnectionId::new(1);
11417 let responses = handler
11418 .handle_control(
11419 conn,
11420 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11421 )
11422 .unwrap();
11423 assert_eq!(responses[0].header.ty, FrameType::HelloAck);
11424 let registration = registry.get_module("aft").unwrap().unwrap();
11425 assert_eq!(registration.control_ops, module_baseline_control_ops());
11426
11427 let frame =
11428 Frame::build(FrameType::Request, control_flags(), 0, 0, 77, Vec::new()).unwrap();
11429 assert!(handler
11430 .guard_module_control_op(&frame, "aft", "route.bind")
11431 .unwrap()
11432 .is_none());
11433 let error = handler
11434 .guard_module_control_op(&frame, "aft", "test.synthetic")
11435 .unwrap()
11436 .expect("synthetic ungranted op should be rejected");
11437 assert_eq!(error.header.ty, FrameType::Error);
11438 assert_eq!(parse_error(&error)["code"], "op_not_allowed");
11439 }
11440
11441 #[test]
11442 fn hello_control_ops_some_adds_optional_grants() {
11443 let registry = Arc::new(Registry::default());
11444 let handler = ControlHandler::new(Arc::clone(®istry));
11445 handler
11446 .handle_control(
11447 ConnectionId::new(1),
11448 hello_frame_with_control_ops(
11449 "aft",
11450 PROTOCOL_VERSION,
11451 7,
11452 Some(vec![
11453 "future.synthetic".to_string(),
11454 "route.bind".to_string(),
11455 ]),
11456 ),
11457 )
11458 .unwrap();
11459 let registration = registry.get_module("aft").unwrap().unwrap();
11460 assert_eq!(
11461 registration.control_ops,
11462 vec![
11463 "route.bind".to_string(),
11464 "route.status".to_string(),
11465 "future.synthetic".to_string(),
11466 ]
11467 );
11468 let frame =
11469 Frame::build(FrameType::Request, control_flags(), 0, 0, 78, Vec::new()).unwrap();
11470 assert!(handler
11471 .guard_module_control_op(&frame, "aft", "future.synthetic")
11472 .unwrap()
11473 .is_none());
11474 }
11475
11476 #[tokio::test]
11477 async fn health_probe_refuses_unadvertised_module_without_sending_frame() {
11478 let registry = Arc::new(Registry::default());
11479 let forwarding = Arc::new(ForwardingTable::default());
11480 let handler =
11481 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11482 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(10));
11483 hello_via_sink(
11484 &handler,
11485 &module_ctx,
11486 &mut module_rx,
11487 hello_frame_with_control_ops("aft", PROTOCOL_VERSION, 7, None),
11488 )
11489 .await;
11490
11491 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(20));
11492 let responses = handler
11493 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(77, "aft"))
11494 .await
11495 .unwrap();
11496 assert_eq!(responses.len(), 1);
11497 assert_eq!(responses[0].header.ty, FrameType::Error);
11498 assert_eq!(parse_error(&responses[0])["code"], "health_not_advertised");
11499 assert!(module_rx.try_recv().is_err());
11500 }
11501
11502 #[tokio::test]
11503 async fn health_probe_demuxes_while_route_bind_relay_is_in_flight() {
11504 let registry = Arc::new(Registry::default());
11505 let forwarding = Arc::new(ForwardingTable::default());
11506 let handler =
11507 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11508 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(30));
11509 hello_via_sink(
11510 &handler,
11511 &module_ctx,
11512 &mut module_rx,
11513 hello_frame_with_control_ops(
11514 "aft",
11515 PROTOCOL_VERSION,
11516 7,
11517 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
11518 ),
11519 )
11520 .await;
11521
11522 let project_root = unique_project_root("demux");
11523 let (route_client_ctx, mut route_client_rx) = route_ctx(ConnectionId::new(31));
11524 let route_handler = handler.clone();
11525 let route_task = tokio::spawn(async move {
11526 route_handler
11527 .handle_control_frame(
11528 &route_client_ctx,
11529 route_open_frame(100, "aft", project_root),
11530 )
11531 .await
11532 .unwrap()
11533 });
11534 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11535 .await
11536 .unwrap()
11537 .unwrap();
11538 assert!(matches!(
11539 serde_json::from_slice::<ModuleControlRequest>(&bind_frame.body).unwrap(),
11540 ModuleControlRequest::RouteBind { .. }
11541 ));
11542
11543 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(32));
11544 let health_handler = handler.clone();
11545 let health_task = tokio::spawn(async move {
11546 health_handler
11547 .handle_control_frame(
11548 &health_client_ctx,
11549 supervisor_health_probe_frame(101, "aft"),
11550 )
11551 .await
11552 .unwrap()
11553 });
11554 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11555 .await
11556 .unwrap()
11557 .unwrap();
11558 assert_eq!(
11559 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
11560 ModuleControlRequest::HealthCheck {}
11561 );
11562
11563 handler
11564 .handle_control_frame(
11565 &module_ctx,
11566 health_response(health_frame.header.corr, HealthStatus::Degraded),
11567 )
11568 .await
11569 .unwrap();
11570 let health_response = health_task.await.unwrap();
11571 assert_eq!(health_response.len(), 1);
11572 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
11573 ClientControlResponse::SupervisorHealthProbe {
11574 module_id,
11575 status,
11576 detail,
11577 metrics,
11578 } => {
11579 assert_eq!(module_id, "aft");
11580 assert_eq!(status, HealthStatus::Degraded);
11581 assert_eq!(detail.as_deref(), Some("warming"));
11582 assert_eq!(metrics, Some(json!({"queue_depth": 3})));
11583 }
11584 other => panic!("unexpected health response: {other:?}"),
11585 }
11586
11587 handler
11588 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
11589 .await
11590 .unwrap();
11591 let route_response = route_task.await.unwrap();
11592 assert!(route_response.is_empty());
11593 let published = route_client_rx.recv().await.unwrap();
11594 assert!(matches!(
11595 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
11596 ClientControlResponse::RouteOpen { .. }
11597 ));
11598 }
11599
11600 async fn relay_route_open(
11605 handler: &ControlHandler,
11606 client_connection: ConnectionId,
11607 client_egress: &FrameSink,
11608 module_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
11609 corr: u64,
11610 module_id: &str,
11611 project_root_label: &str,
11612 ) -> (tokio::task::JoinHandle<Vec<Frame>>, Frame) {
11613 let ctx = RouteCtx {
11614 connection_id: client_connection,
11615 egress: client_egress.clone(),
11616 };
11617 let handler = handler.clone();
11618 let project_root = unique_project_root(project_root_label);
11619 let module_id = module_id.to_string();
11620 let dispatch = tracing::dispatcher::get_default(|dispatch| dispatch.clone());
11621 let task = tokio::spawn(async move {
11622 let _guard = tracing::dispatcher::set_default(&dispatch);
11623 handler
11624 .handle_control_frame(&ctx, route_open_frame(corr, &module_id, project_root))
11625 .await
11626 .unwrap()
11627 });
11628 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
11629 .await
11630 .expect("module receives the relayed route.bind")
11631 .expect("module egress is open");
11632 (task, bind.frame)
11633 }
11634
11635 fn route_bind_channel(frame: &Frame) -> (u16, u32) {
11636 match serde_json::from_slice::<ModuleControlRequest>(&frame.body).unwrap() {
11637 ModuleControlRequest::RouteBind {
11638 route_channel,
11639 epoch,
11640 ..
11641 } => (route_channel, epoch),
11642 other => panic!("expected a route.bind request, got {other:?}"),
11643 }
11644 }
11645
11646 fn published_route(frame: &Frame) -> (u16, u32) {
11647 match serde_json::from_slice::<ClientControlResponse>(&frame.body).unwrap() {
11648 ClientControlResponse::RouteOpen {
11649 route_channel,
11650 route_epoch,
11651 ..
11652 } => (route_channel, route_epoch),
11653 other => panic!("expected a route.open response, got {other:?}"),
11654 }
11655 }
11656
11657 #[tokio::test]
11677 async fn late_bind_ack_for_a_closing_client_keeps_the_module_connection_serving() {
11678 let registry = Arc::new(Registry::default());
11679 let forwarding = Arc::new(ForwardingTable::default());
11680 let handler =
11681 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
11682
11683 let module_connection = ConnectionId::new(30);
11684 let (module_ctx, mut module_rx) = route_ctx(module_connection);
11685 hello_via_sink(
11686 &handler,
11687 &module_ctx,
11688 &mut module_rx,
11689 hello_frame("aft", PROTOCOL_VERSION, 7),
11690 )
11691 .await;
11692
11693 let dying_client = ConnectionId::new(31);
11694 let (dying_ctx, mut dying_rx) = route_ctx(dying_client);
11695
11696 let (first_task, first_bind) = relay_route_open(
11699 &handler,
11700 dying_client,
11701 &dying_ctx.egress,
11702 &mut module_rx,
11703 100,
11704 "aft",
11705 "closing-first",
11706 )
11707 .await;
11708 handler
11709 .handle_control_frame(&module_ctx, route_bind_ack(first_bind.header.corr))
11710 .await
11711 .unwrap();
11712 assert!(first_task.await.unwrap().is_empty());
11713 let (first_channel, first_epoch) = published_route(&dying_rx.recv().await.unwrap());
11714
11715 let (second_task, second_bind) = relay_route_open(
11717 &handler,
11718 dying_client,
11719 &dying_ctx.egress,
11720 &mut module_rx,
11721 101,
11722 "aft",
11723 "closing-second",
11724 )
11725 .await;
11726 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&second_bind);
11727
11728 assert!(forwarding
11731 .escalate_client_delivery_failure(
11732 dying_client,
11733 first_channel,
11734 first_epoch,
11735 CloseReason::new(
11736 "module_to_client_delivery_failed",
11737 "client egress refused a module frame",
11738 ),
11739 crate::forwarding::UndeliveredFrame {
11740 module_id: None,
11741 sink: &dying_ctx.egress,
11742 },
11743 )
11744 .unwrap());
11745 assert!(!dying_ctx.egress.is_closed());
11746
11747 let ack = handler
11749 .handle_control_frame(&module_ctx, route_bind_ack(second_bind.header.corr))
11750 .await;
11751 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
11752 if module_loop_error.is_some() {
11753 handler.cleanup_connection(module_connection).unwrap();
11757 }
11758 let post_ack_module_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
11762 .await
11763 .ok()
11764 .flatten();
11765
11766 assert!(
11768 registry
11769 .get_module_by_connection(module_connection)
11770 .unwrap()
11771 .is_some(),
11772 "one client's closing connection ended the shared module connection: \
11773 {module_loop_error:?}"
11774 );
11775 let cotenant = ConnectionId::new(32);
11777 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
11778 let (cotenant_task, cotenant_bind) = relay_route_open(
11779 &handler,
11780 cotenant,
11781 &cotenant_ctx.egress,
11782 &mut module_rx,
11783 102,
11784 "aft",
11785 "closing-cotenant",
11786 )
11787 .await;
11788 handler
11789 .handle_control_frame(&module_ctx, route_bind_ack(cotenant_bind.header.corr))
11790 .await
11791 .unwrap();
11792 assert!(cotenant_task.await.unwrap().is_empty());
11793 let (cotenant_channel, cotenant_epoch) =
11794 published_route(&cotenant_rx.recv().await.unwrap());
11795 assert!(matches!(
11796 forwarding
11797 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
11798 .unwrap(),
11799 DataRoute::Client(DataRouteState::Bound(_))
11800 ));
11801
11802 let goodbye = post_ack_module_frame
11805 .expect("module receives a GOODBYE for the abandoned route channel");
11806 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
11807 assert_eq!(goodbye.header.channel, abandoned_channel);
11808 assert_eq!(goodbye.header.epoch, abandoned_epoch);
11809
11810 assert!(dying_rx.try_recv().is_err());
11814 let second_response = second_task.await.unwrap();
11815 assert_eq!(second_response.len(), 1);
11816 assert_eq!(
11817 parse_error(&second_response[0])["code"],
11818 "target_unavailable"
11819 );
11820 }
11821
11822 async fn catalog_snapshot(
11825 handler: &ControlHandler,
11826 client_ctx: &RouteCtx,
11827 corr: u64,
11828 ) -> (u64, Vec<(String, bool)>) {
11829 let body =
11830 serde_json::to_vec(&ClientControlRequest::CatalogList { module_id: None }).unwrap();
11831 let request = Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap();
11832 let reply = handler
11833 .handle_control_frame(client_ctx, request)
11834 .await
11835 .unwrap()
11836 .pop()
11837 .unwrap();
11838 let ClientControlResponse::CatalogList {
11839 generation,
11840 modules,
11841 ..
11842 } = serde_json::from_slice(&reply.body).unwrap()
11843 else {
11844 panic!("not a catalog.list reply");
11845 };
11846 (
11847 generation,
11848 modules
11849 .into_iter()
11850 .map(|entry| (entry.module_id, entry.ready))
11851 .collect(),
11852 )
11853 }
11854
11855 fn hello_held_before_forwarding() -> (
11861 ControlHandler,
11862 Arc<std::sync::Barrier>,
11863 std::thread::JoinHandle<Vec<Frame>>,
11864 RouteCtx,
11865 mpsc::Receiver<crate::router::OutboundFrame>,
11866 ) {
11867 let hold = Arc::new(std::sync::Barrier::new(2));
11868 let handler = ControlHandler::with_forwarding(
11869 Arc::new(Registry::default()),
11870 Arc::new(ForwardingTable::default()),
11871 )
11872 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
11873 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
11874 let hello = {
11875 let handler = handler.clone();
11876 let module_ctx = module_ctx.clone();
11877 std::thread::spawn(move || {
11878 handler
11879 .handle_hello(
11880 module_ctx.connection_id,
11881 Some(module_ctx.egress.clone()),
11882 hello_frame("aft", PROTOCOL_VERSION, 7),
11883 )
11884 .unwrap()
11885 })
11886 };
11887 (handler, hold, hello, module_ctx, module_rx)
11888 }
11889
11890 #[tokio::test]
11899 async fn module_is_listed_only_once_route_open_to_it_can_succeed() {
11900 let (handler, hold, hello, module_ctx, mut module_rx) = hello_held_before_forwarding();
11901 let client = ConnectionId::new(41);
11902 let (client_ctx, mut client_rx) = route_ctx(client);
11903
11904 hold.wait();
11905 assert!(handler.registry.get_module("aft").unwrap().is_some());
11907 assert!(!handler
11908 .forwarding
11909 .has_live_module_connection("aft")
11910 .unwrap());
11911 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 300).await;
11912 assert!(
11913 listed.is_empty(),
11914 "catalog.list listed a module route.open cannot reach yet: {listed:?}"
11915 );
11916 let refused = handler
11917 .handle_control_frame(
11918 &client_ctx,
11919 route_open_frame(301, "aft", unique_project_root("listed-window")),
11920 )
11921 .await
11922 .unwrap();
11923 assert_eq!(parse_error(&refused[0])["code"], "target_unavailable");
11924 let (duplicate_ctx, _duplicate_rx) = route_ctx(ConnectionId::new(42));
11925 let duplicate = handler
11926 .handle_control_frame(&duplicate_ctx, hello_frame("aft", PROTOCOL_VERSION, 8))
11927 .await
11928 .unwrap();
11929 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
11930
11931 hold.wait();
11932 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11933 let ack = module_rx.try_recv().unwrap().frame;
11934 assert_eq!(ack.header.ty, FrameType::HelloAck);
11935 let (_, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11936 assert_eq!(listed, vec![("aft".to_string(), true)]);
11937
11938 let (open_task, bind) = relay_route_open(
11939 &handler,
11940 client,
11941 &client_ctx.egress,
11942 &mut module_rx,
11943 303,
11944 "aft",
11945 "listed-after",
11946 )
11947 .await;
11948 assert!(handler
11949 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
11950 .await
11951 .unwrap()
11952 .is_empty());
11953 assert!(open_task.await.unwrap().is_empty(), "route.open accepted");
11954 published_route(&client_rx.try_recv().unwrap().frame);
11955 }
11956
11957 #[tokio::test]
11964 async fn catalog_generation_advances_when_the_module_appears_and_not_before() {
11965 let (handler, hold, hello, _module_ctx, _module_rx) = hello_held_before_forwarding();
11966 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
11967 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
11968
11969 hold.wait();
11970 let (during, listed) = catalog_snapshot(&handler, &client_ctx, 301).await;
11971 assert!(listed.is_empty());
11972 assert_eq!(
11973 during, before,
11974 "the generation moved while the module was not listed"
11975 );
11976
11977 hold.wait();
11978 assert!(hello.join().unwrap().is_empty(), "HELLO succeeds");
11979 let (after, listed) = catalog_snapshot(&handler, &client_ctx, 302).await;
11980 assert_eq!(listed, vec![("aft".to_string(), true)]);
11981 assert_eq!(
11982 after,
11983 before + 1,
11984 "the module appeared without exactly one generation bump"
11985 );
11986 }
11987
11988 #[tokio::test]
11993 async fn failed_forwarding_registration_leaves_no_registry_or_catalog_trace() {
11994 let registry = Arc::new(Registry::default());
11995 let handler = ControlHandler::with_forwarding(
11996 Arc::clone(®istry),
11997 Arc::new(ForwardingTable::default()),
11998 );
11999 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
12000 let (before, _) = catalog_snapshot(&handler, &client_ctx, 300).await;
12001 let (module_ctx, module_rx) = route_ctx(ConnectionId::new(40));
12002 drop(module_rx);
12003
12004 let replies = handler
12005 .handle_control_frame(&module_ctx, hello_frame("aft", PROTOCOL_VERSION, 7))
12006 .await
12007 .unwrap();
12008 assert_eq!(replies[0].header.ty, FrameType::Error, "{replies:?}");
12009 assert!(registry.get_module("aft").unwrap().is_none());
12010 assert!(registry
12011 .get_module_by_connection(module_ctx.connection_id)
12012 .unwrap()
12013 .is_none());
12014 assert_eq!(
12015 catalog_snapshot(&handler, &client_ctx, 301).await,
12016 (before, Vec::new())
12017 );
12018 }
12019
12020 async fn route_open_held_after_bind_send() -> (
12024 ControlHandler,
12025 Arc<tokio::sync::Barrier>,
12026 RouteCtx,
12027 mpsc::Receiver<crate::router::OutboundFrame>,
12028 ) {
12029 let hold = Arc::new(tokio::sync::Barrier::new(2));
12030 let handler = ControlHandler::with_forwarding(
12031 Arc::new(Registry::default()),
12032 Arc::new(ForwardingTable::default()),
12033 )
12034 .with_route_bind_relay_sent_hold(Arc::clone(&hold));
12035 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
12036 hello_via_sink(
12037 &handler,
12038 &module_ctx,
12039 &mut module_rx,
12040 hello_frame("aft", PROTOCOL_VERSION, 7),
12041 )
12042 .await;
12043 (handler, hold, module_ctx, module_rx)
12044 }
12045
12046 #[tokio::test]
12053 async fn bind_acked_before_the_enqueue_mark_commits_without_a_module_goodbye() {
12054 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12055 let client = ConnectionId::new(41);
12056 let (client_ctx, mut client_rx) = route_ctx(client);
12057
12058 let (open_task, bind) = relay_route_open(
12059 &handler,
12060 client,
12061 &client_ctx.egress,
12062 &mut module_rx,
12063 200,
12064 "aft",
12065 "acked-before-mark",
12066 )
12067 .await;
12068 let (module_channel, module_epoch) = route_bind_channel(&bind);
12069 hold.wait().await;
12071
12072 assert!(handler
12074 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12075 .await
12076 .unwrap()
12077 .is_empty());
12078 let (client_channel, client_epoch) = published_route(&client_rx.try_recv().unwrap().frame);
12079
12080 hold.wait().await;
12081 assert!(
12082 open_task.await.unwrap().is_empty(),
12083 "route.open answers accepted"
12084 );
12085
12086 match module_rx.try_recv() {
12089 Err(mpsc::error::TryRecvError::Empty) => {}
12090 Ok(outbound) => panic!(
12091 "module was sent {:?} on channel {} epoch {} for a route the daemon committed",
12092 outbound.frame.header.ty,
12093 outbound.frame.header.channel,
12094 outbound.frame.header.epoch
12095 ),
12096 Err(err) => panic!("module egress closed: {err}"),
12097 }
12098
12099 let router = Router::with_control_handler(Arc::new(handler.clone()));
12102 let request = Frame::build(
12103 FrameType::Request,
12104 Flags::new(true, Priority::Interactive, false),
12105 client_channel,
12106 client_epoch,
12107 201,
12108 b"first request".to_vec(),
12109 )
12110 .unwrap();
12111 router
12112 .route_for_connection(&client_ctx, request)
12113 .await
12114 .unwrap();
12115 let delivered = module_rx.try_recv().unwrap().frame;
12116 assert_eq!(delivered.header.ty, FrameType::Request);
12117 assert_eq!(
12118 (delivered.header.channel, delivered.header.epoch),
12119 (module_channel, module_epoch)
12120 );
12121 assert_eq!(delivered.body, b"first request");
12122 }
12123
12124 #[tokio::test]
12130 async fn bind_released_before_the_enqueue_mark_still_sends_the_module_a_goodbye() {
12131 let (handler, hold, _module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12132 let client = ConnectionId::new(42);
12133 let (client_ctx, _client_rx) = route_ctx(client);
12134
12135 let (open_task, bind) = relay_route_open(
12136 &handler,
12137 client,
12138 &client_ctx.egress,
12139 &mut module_rx,
12140 300,
12141 "aft",
12142 "released-before-mark",
12143 )
12144 .await;
12145 let (module_channel, module_epoch) = route_bind_channel(&bind);
12146 hold.wait().await;
12147
12148 handler.cleanup_connection(client).unwrap();
12149 assert!(
12150 module_rx.try_recv().is_err(),
12151 "client teardown must not GOODBYE a bind route.open is still queueing"
12152 );
12153
12154 hold.wait().await;
12155 let response = open_task.await.unwrap();
12156 assert_eq!(response.len(), 1);
12157 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12158
12159 let goodbye = module_rx
12160 .try_recv()
12161 .expect("module is told to drop the binding the daemon abandoned")
12162 .frame;
12163 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12164 assert_eq!(
12165 (goodbye.header.channel, goodbye.header.epoch),
12166 (module_channel, module_epoch)
12167 );
12168 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
12169 }
12170
12171 #[tokio::test]
12176 async fn bind_acked_for_a_gone_client_before_the_enqueue_mark_sends_exactly_one_goodbye() {
12177 let (handler, hold, module_ctx, mut module_rx) = route_open_held_after_bind_send().await;
12178 let client = ConnectionId::new(43);
12179 let (client_ctx, client_rx) = route_ctx(client);
12180
12181 let (open_task, bind) = relay_route_open(
12182 &handler,
12183 client,
12184 &client_ctx.egress,
12185 &mut module_rx,
12186 400,
12187 "aft",
12188 "gone-client-before-mark",
12189 )
12190 .await;
12191 let (module_channel, module_epoch) = route_bind_channel(&bind);
12192 hold.wait().await;
12193
12194 drop(client_rx);
12195 assert!(client_ctx.egress.is_closed());
12196 handler
12197 .handle_control_frame(&module_ctx, route_bind_ack(bind.header.corr))
12198 .await
12199 .unwrap();
12200
12201 hold.wait().await;
12202 let response = open_task.await.unwrap();
12203 assert_eq!(response.len(), 1);
12204 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
12205
12206 let goodbye = module_rx
12207 .try_recv()
12208 .expect("module is told to drop the binding it acked")
12209 .frame;
12210 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
12211 assert_eq!(
12212 (goodbye.header.channel, goodbye.header.epoch),
12213 (module_channel, module_epoch)
12214 );
12215 assert!(module_rx.try_recv().is_err(), "exactly one GOODBYE");
12216 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
12217 }
12218
12219 #[test]
12226 fn only_the_modules_own_closing_connection_ends_the_module_loop() {
12227 let handler = ControlHandler::default();
12228 let module_connection = ConnectionId::new(30);
12229 let client_connection = ConnectionId::new(31);
12230
12231 handler
12232 .refuse_to_end_module_connection_for_a_client(
12233 module_connection,
12234 77,
12235 ForwardingError::ConnectionClosing {
12236 connection_id: client_connection,
12237 },
12238 )
12239 .expect("a closing client must never end the module connection");
12240
12241 assert!(matches!(
12242 handler.refuse_to_end_module_connection_for_a_client(
12243 module_connection,
12244 78,
12245 ForwardingError::ConnectionClosing {
12246 connection_id: module_connection,
12247 },
12248 ),
12249 Err(RouterError::Forwarding(ForwardingError::ConnectionClosing {
12250 connection_id
12251 })) if connection_id == module_connection
12252 ));
12253 assert!(matches!(
12254 handler.refuse_to_end_module_connection_for_a_client(
12255 module_connection,
12256 79,
12257 ForwardingError::Poisoned,
12258 ),
12259 Err(RouterError::Forwarding(ForwardingError::Poisoned))
12260 ));
12261 assert!(matches!(
12262 handler.refuse_to_end_module_connection_for_a_client(
12263 module_connection,
12264 80,
12265 ForwardingError::StaleModuleEndpoint,
12266 ),
12267 Err(RouterError::Forwarding(
12268 ForwardingError::StaleModuleEndpoint
12269 ))
12270 ));
12271 }
12272
12273 #[tokio::test]
12286 async fn route_open_refuses_consumer_identity_that_fails_spawn_attestation() {
12287 let registry = Arc::new(Registry::default());
12288 let forwarding = Arc::new(ForwardingTable::default());
12289 let supervisor = SupervisorHandle::new();
12290 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12291 let handler =
12292 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12293 .with_supervisor(supervisor);
12294
12295 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12296 hello_via_sink(
12297 &handler,
12298 &target_ctx,
12299 &mut target_rx,
12300 hello_frame("target", PROTOCOL_VERSION, 1),
12301 )
12302 .await;
12303
12304 let wrong_nonce = handler
12308 .handle_control_frame(
12309 &route_ctx(ConnectionId::new(91)).0,
12310 route_open_frame_with_admission_facts(
12311 20,
12312 "target",
12313 unique_project_root("admission-facts"),
12314 Some(subc_control::ConsumerIdentity {
12315 module_id: "fed".to_string(),
12316 launch_nonce: "not-the-real-nonce".to_string(),
12317 }),
12318 None,
12319 ),
12320 )
12321 .await
12322 .unwrap();
12323 assert_eq!(
12324 parse_error(&wrong_nonce[0])["code"],
12325 "bad_consumer_identity",
12326 "a mismatched launch nonce must be refused, not stamped Reserved"
12327 );
12328
12329 let never_spawned = handler
12333 .handle_control_frame(
12334 &route_ctx(ConnectionId::new(92)).0,
12335 route_open_frame_with_admission_facts(
12336 21,
12337 "target",
12338 unique_project_root("admission-facts"),
12339 Some(subc_control::ConsumerIdentity {
12340 module_id: "never-spawned".to_string(),
12341 launch_nonce: "any-nonce".to_string(),
12342 }),
12343 None,
12344 ),
12345 )
12346 .await
12347 .unwrap();
12348 assert_eq!(
12349 parse_error(&never_spawned[0])["code"],
12350 "bad_consumer_identity",
12351 "an unspawned module_id must be refused rather than accepted for lack of a record"
12352 );
12353 }
12354
12355 #[tokio::test]
12378 async fn route_open_stamps_reserved_for_a_correctly_attested_consumer() {
12379 let registry = Arc::new(Registry::default());
12380 let forwarding = Arc::new(ForwardingTable::default());
12381 let supervisor = SupervisorHandle::new();
12382 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12383 let handler =
12384 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12385 .with_supervisor(supervisor);
12386
12387 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(95));
12388 hello_via_sink(
12389 &handler,
12390 &target_ctx,
12391 &mut target_rx,
12392 hello_frame("target", PROTOCOL_VERSION, 1),
12393 )
12394 .await;
12395
12396 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(96));
12397 let route_handler = handler.clone();
12398 let route_task = tokio::spawn(async move {
12399 route_handler
12400 .handle_control_frame(
12401 &client_ctx,
12402 route_open_frame_with_admission_facts(
12403 30,
12404 "target",
12405 unique_project_root("admission-facts"),
12406 Some(subc_control::ConsumerIdentity {
12407 module_id: "fed".to_string(),
12408 launch_nonce: "fed-nonce".to_string(),
12409 }),
12410 None,
12411 ),
12412 )
12413 .await
12414 .unwrap()
12415 });
12416
12417 let bind_frame = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12424 .await
12425 .expect("no route.bind within 5s: the consumer-identity guard refused a correctly attested consumer")
12426 .expect("module control channel closed before route.bind");
12427 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
12428 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
12429 panic!("expected route.bind")
12430 };
12431 assert_eq!(
12432 principal,
12433 Some(Principal::Reserved {
12434 module_id: "fed".to_string()
12435 }),
12436 "a correctly attested consumer must be stamped Reserved for its own id"
12437 );
12438
12439 handler
12440 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
12441 .await
12442 .unwrap();
12443 assert!(route_task.await.unwrap().is_empty());
12444 assert!(
12445 matches!(
12446 serde_json::from_slice::<ClientControlResponse>(
12447 &client_rx.recv().await.unwrap().body
12448 )
12449 .unwrap(),
12450 ClientControlResponse::RouteOpen { .. }
12451 ),
12452 "the route must actually open, not merely avoid an error"
12453 );
12454 }
12455
12456 #[tokio::test(start_paused = true)]
12457 async fn supervisor_routes_serializes_live_draining_bindings_from_the_real_handler() {
12458 let registry = Arc::new(Registry::default());
12459 let forwarding = Arc::new(ForwardingTable::default());
12460 let supervisor = SupervisorHandle::new();
12461 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12462 let handler =
12463 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12464 .with_supervisor(supervisor);
12465
12466 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(101));
12467 hello_via_sink(
12468 &handler,
12469 &target_ctx,
12470 &mut target_rx,
12471 hello_frame("target", PROTOCOL_VERSION, 1),
12472 )
12473 .await;
12474
12475 let (direct_ctx, mut direct_rx) = route_ctx(ConnectionId::new(102));
12476 let direct_handler = handler.clone();
12477 let direct_open = tokio::spawn(async move {
12478 direct_handler
12479 .handle_control_frame(
12480 &direct_ctx,
12481 route_open_frame(2, "target", unique_project_root("route-census-direct")),
12482 )
12483 .await
12484 .unwrap()
12485 });
12486 let direct_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12487 .await
12488 .expect("no direct route.bind within 5s")
12489 .expect("target control channel closed before direct route.bind");
12490 handler
12491 .handle_control_frame(&target_ctx, route_bind_ack(direct_bind.header.corr))
12492 .await
12493 .unwrap();
12494 assert!(direct_open.await.unwrap().is_empty());
12495 let _ = direct_rx.recv().await.unwrap();
12496
12497 let (reserved_ctx, mut reserved_rx) = route_ctx(ConnectionId::new(103));
12498 let reserved_handler = handler.clone();
12499 let reserved_open = tokio::spawn(async move {
12500 reserved_handler
12501 .handle_control_frame(
12502 &reserved_ctx,
12503 route_open_frame_with_admission_facts(
12504 3,
12505 "target",
12506 unique_project_root("admission-facts"),
12507 Some(ConsumerIdentity {
12508 module_id: "fed".to_string(),
12509 launch_nonce: "fed-nonce".to_string(),
12510 }),
12511 None,
12512 ),
12513 )
12514 .await
12515 .unwrap()
12516 });
12517 let reserved_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12518 .await
12519 .expect("no reserved route.bind within 5s")
12520 .expect("target control channel closed before reserved route.bind");
12521 handler
12522 .handle_control_frame(&target_ctx, route_bind_ack(reserved_bind.header.corr))
12523 .await
12524 .unwrap();
12525 assert!(reserved_open.await.unwrap().is_empty());
12526 let _ = reserved_rx.recv().await.unwrap();
12527
12528 forwarding
12529 .begin_module_drain("target", subc_control::RouteCloseReason::Reload)
12530 .unwrap();
12531 let (census_ctx, _census_rx) = route_ctx(ConnectionId::new(104));
12532 let census_body = serde_json::to_vec(&ClientControlRequest::SupervisorRoutes {
12533 module_id: Some("target".to_string()),
12534 })
12535 .unwrap();
12536 let census_frame =
12537 Frame::build(FrameType::Request, control_flags(), 0, 0, 4, census_body).unwrap();
12538 let response = handler
12539 .handle_control_frame(&census_ctx, census_frame)
12540 .await
12541 .unwrap()
12542 .pop()
12543 .unwrap();
12544 let actual: Value = serde_json::from_slice(&response.body).unwrap();
12545 let decoded: ClientControlResponse = serde_json::from_value(actual.clone()).unwrap();
12546 assert!(matches!(
12547 decoded,
12548 ClientControlResponse::SupervisorRoutes { .. }
12549 ));
12550 let routes = actual["modules"][0]["routes"].as_array().unwrap();
12551 assert_eq!(routes.len(), 2);
12552 assert!(routes.iter().all(|route| route["draining"] == true));
12553 assert!(
12556 routes.iter().all(|route| route["drain_reason"] == "reload"),
12557 "draining routes must name the drain's reason: {routes:?}"
12558 );
12559 assert!(routes.iter().any(|route| {
12560 route["consumer"] == serde_json::json!({"kind": "direct", "connection_id": 102})
12561 }));
12562 assert!(routes.iter().any(|route| {
12563 route["consumer"] == serde_json::json!({"kind": "reserved", "module_id": "fed"})
12564 }));
12565
12566 let golden_path = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12567 .join("../subc-control/tests/golden/client_control_response_supervisor_routes.json");
12568 if std::env::var_os("UPDATE_GOLDEN").is_some() {
12569 std::fs::write(
12570 &golden_path,
12571 format!("{}\n", serde_json::to_string_pretty(&actual).unwrap()),
12572 )
12573 .unwrap();
12574 }
12575 let expected: Value =
12576 serde_json::from_str(&std::fs::read_to_string(golden_path).unwrap()).unwrap();
12577 assert_eq!(actual, expected);
12578 }
12579
12580 async fn query_live_roots(
12581 handler: &ControlHandler,
12582 module_ctx: &RouteCtx,
12583 ) -> ModuleControlResponseToModule {
12584 let body = serde_json::to_vec(&ModuleControlRequestFromModule::LiveRoots {}).unwrap();
12585 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 900, body).unwrap();
12586 let response = handler
12587 .handle_control_frame(module_ctx, frame)
12588 .await
12589 .unwrap()
12590 .pop()
12591 .unwrap();
12592 serde_json::from_slice(&response.body).unwrap()
12593 }
12594
12595 #[tokio::test(start_paused = true)]
12596 async fn supervisor_live_roots_root_known_arm_counts_bound_and_pending_from_real_handler() {
12597 let registry = Arc::new(Registry::default());
12598 let forwarding = Arc::new(ForwardingTable::default());
12599 let handler = ControlHandler::with_forwarding(registry, forwarding);
12600 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(301));
12601 hello_via_sink(
12602 &handler,
12603 &target_ctx,
12604 &mut target_rx,
12605 hello_frame("target", PROTOCOL_VERSION, 1),
12606 )
12607 .await;
12608 let root = unique_project_root("live-roots-known");
12609 let path = ProjectRootId::from_path_allowing_missing(root.path())
12610 .unwrap()
12611 .as_path()
12612 .to_path_buf();
12613 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(302));
12614 let open_handler = handler.clone();
12615 let opened = tokio::spawn(async move {
12616 open_handler
12617 .handle_control_frame(&client_ctx, legacy_route_open_frame(2, "target", root))
12618 .await
12619 .unwrap()
12620 });
12621 let bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12622 .await
12623 .unwrap()
12624 .unwrap();
12625 handler
12626 .handle_control_frame(&target_ctx, route_bind_ack(bind.header.corr))
12627 .await
12628 .unwrap();
12629 assert!(opened.await.unwrap().is_empty());
12630 let _ = client_rx.recv().await.unwrap();
12631
12632 let root = unique_project_root("live-roots-pending");
12633 let pending_path = ProjectRootId::from_path_allowing_missing(root.path())
12634 .unwrap()
12635 .as_path()
12636 .to_path_buf();
12637 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(303));
12638 let open_handler = handler.clone();
12639 let pending = tokio::spawn(async move {
12640 open_handler
12641 .handle_control_frame(&client_ctx, legacy_route_open_frame(3, "target", root))
12642 .await
12643 .unwrap()
12644 });
12645 let pending_bind = tokio::time::timeout(Duration::from_secs(5), target_rx.recv())
12646 .await
12647 .unwrap()
12648 .unwrap();
12649 let actual = query_live_roots(&handler, &target_ctx).await;
12650 let ModuleControlResponseToModule::LiveRoots {
12651 roots,
12652 unknown_root_bindings,
12653 total_bindings,
12654 } = actual
12655 else {
12656 panic!("expected live roots")
12657 };
12658 assert_eq!(total_bindings, 2, "root-known arm must count live routes");
12659 assert_eq!(unknown_root_bindings, 0);
12660 assert_eq!(
12661 roots.len(),
12662 2,
12663 "root-known arm must retain each canonical root"
12664 );
12665 assert_eq!(
12666 total_bindings,
12667 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12668 );
12669 let counts = roots
12670 .iter()
12671 .map(|root| (root.project_root.clone(), root.bound, root.pending))
12672 .collect::<Vec<_>>();
12673 let mut expected = vec![(path, 1, 0), (pending_path, 0, 1)];
12674 expected.sort_by(|a, b| a.0.cmp(&b.0));
12675 assert_eq!(
12676 counts, expected,
12677 "roots must sort by path and count pending separately"
12678 );
12679 handler
12680 .handle_control_frame(&target_ctx, route_bind_ack(pending_bind.header.corr))
12681 .await
12682 .unwrap();
12683 assert!(pending.await.unwrap().is_empty());
12684 }
12685
12686 #[tokio::test(start_paused = true)]
12687 async fn supervisor_live_roots_unknown_root_arm_is_not_no_bindings() {
12688 let forwarding = Arc::new(ForwardingTable::default());
12689 let handler =
12690 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12691 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(311));
12692 hello_via_sink(
12693 &handler,
12694 &target_ctx,
12695 &mut target_rx,
12696 hello_frame("target", PROTOCOL_VERSION, 1),
12697 )
12698 .await;
12699 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(312));
12700 let pending = forwarding
12701 .begin_route_bind_relay_for_test(
12702 client_ctx.connection_id,
12703 client_ctx.egress.clone(),
12704 2,
12705 "target",
12706 )
12707 .unwrap();
12708 forwarding
12709 .complete_pending_relay(
12710 target_ctx.connection_id,
12711 pending.corr,
12712 RouteBindRelayOutcome::Accepted,
12713 )
12714 .unwrap();
12715 let actual = query_live_roots(&handler, &target_ctx).await;
12716 let ModuleControlResponseToModule::LiveRoots {
12717 roots,
12718 unknown_root_bindings,
12719 total_bindings,
12720 } = actual
12721 else {
12722 panic!("expected live roots")
12723 };
12724 assert!(roots.is_empty(), "unknown-root arm must not invent a root");
12725 assert_eq!(
12726 unknown_root_bindings, 1,
12727 "unknown-root arm must not read as no bindings"
12728 );
12729 assert_eq!(total_bindings, 1, "unknown-root arm has a live binding");
12730 assert_eq!(
12731 total_bindings,
12732 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12733 );
12734 }
12735
12736 #[tokio::test(start_paused = true)]
12744 async fn hello_ack_reaches_the_module_before_a_route_bind_raced_into_the_reply_gap() {
12745 let forwarding = Arc::new(ForwardingTable::default());
12746 let handler =
12747 ControlHandler::with_forwarding(Arc::new(Registry::default()), Arc::clone(&forwarding));
12748 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(341));
12749 let replies = handler
12750 .handle_control_frame(&module_ctx, hello_frame("raced", PROTOCOL_VERSION, 7))
12751 .await
12752 .unwrap();
12753 let queued_by_hello = module_rx.len();
12754
12755 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(342));
12756 let open_handler = handler.clone();
12757 let open = tokio::spawn(async move {
12758 open_handler
12759 .handle_control_frame(
12760 &client_ctx,
12761 route_open_frame(2, "raced", unique_project_root("hello-ack-race")),
12762 )
12763 .await
12764 .unwrap()
12765 });
12766 let mut spins = 0;
12768 while module_rx.len() == queued_by_hello {
12769 spins += 1;
12770 assert!(spins < 10_000, "route.open never queued a route.bind");
12771 tokio::task::yield_now().await;
12772 }
12773
12774 for reply in replies {
12776 module_ctx.egress.send(reply).await.unwrap();
12777 }
12778
12779 let first = module_rx.recv().await.unwrap().frame;
12780 assert_eq!(
12781 first.header.ty,
12782 FrameType::HelloAck,
12783 "the first frame a registering module reads must be its HELLO_ACK"
12784 );
12785 assert_eq!(first.header.corr, 7);
12786 let second = module_rx.recv().await.unwrap().frame;
12787 assert_eq!(second.header.ty, FrameType::Request);
12788 assert!(
12789 matches!(
12790 serde_json::from_slice::<ModuleControlRequest>(&second.body).unwrap(),
12791 ModuleControlRequest::RouteBind { .. }
12792 ),
12793 "the route.bind follows the ack"
12794 );
12795 assert!(module_rx.try_recv().is_err(), "nothing else was queued");
12796
12797 handler
12798 .handle_control_frame(&module_ctx, route_bind_ack(second.header.corr))
12799 .await
12800 .unwrap();
12801 assert!(open.await.unwrap().is_empty());
12802 let _ = client_rx.recv().await.unwrap();
12803 }
12804
12805 #[tokio::test(start_paused = true)]
12806 async fn supervisor_live_roots_cross_module_scope_uses_requesting_connection() {
12807 let handler = ControlHandler::with_forwarding(
12808 Arc::new(Registry::default()),
12809 Arc::new(ForwardingTable::default()),
12810 );
12811 let (first_ctx, mut first_rx) = route_ctx(ConnectionId::new(315));
12812 let (second_ctx, mut second_rx) = route_ctx(ConnectionId::new(316));
12813 hello_via_sink(
12814 &handler,
12815 &first_ctx,
12816 &mut first_rx,
12817 hello_frame("first", PROTOCOL_VERSION, 1),
12818 )
12819 .await;
12820 hello_via_sink(
12821 &handler,
12822 &second_ctx,
12823 &mut second_rx,
12824 hello_frame("second", PROTOCOL_VERSION, 2),
12825 )
12826 .await;
12827 let root = unique_project_root("second-only");
12828 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(317));
12829 let cloned = handler.clone();
12830 let open = tokio::spawn(async move {
12831 cloned
12832 .handle_control_frame(&client_ctx, route_open_frame(3, "second", root))
12833 .await
12834 .unwrap()
12835 });
12836 let bind = tokio::time::timeout(Duration::from_secs(5), second_rx.recv())
12837 .await
12838 .unwrap()
12839 .unwrap();
12840 let first = query_live_roots(&handler, &first_ctx).await;
12841 let second = query_live_roots(&handler, &second_ctx).await;
12842 assert!(
12843 matches!(
12844 first,
12845 ModuleControlResponseToModule::LiveRoots {
12846 total_bindings: 0,
12847 ..
12848 }
12849 ),
12850 "cross-module scope must not expose another module's roots"
12851 );
12852 assert!(
12853 matches!(
12854 second,
12855 ModuleControlResponseToModule::LiveRoots {
12856 total_bindings: 1,
12857 ..
12858 }
12859 ),
12860 "second module must see its pending route"
12861 );
12862 handler
12863 .handle_control_frame(&second_ctx, route_bind_ack(bind.header.corr))
12864 .await
12865 .unwrap();
12866 assert!(open.await.unwrap().is_empty());
12867 }
12868
12869 #[tokio::test(start_paused = true)]
12870 async fn supervisor_live_roots_no_bindings_arm_is_empty() {
12871 let handler = ControlHandler::with_forwarding(
12872 Arc::new(Registry::default()),
12873 Arc::new(ForwardingTable::default()),
12874 );
12875 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(321));
12876 hello_via_sink(
12877 &handler,
12878 &target_ctx,
12879 &mut target_rx,
12880 hello_frame("target", PROTOCOL_VERSION, 1),
12881 )
12882 .await;
12883 let actual = query_live_roots(&handler, &target_ctx).await;
12884 let ModuleControlResponseToModule::LiveRoots {
12885 roots,
12886 unknown_root_bindings,
12887 total_bindings,
12888 } = actual
12889 else {
12890 panic!("expected live roots")
12891 };
12892 assert!(roots.is_empty());
12893 assert_eq!(unknown_root_bindings, 0);
12894 assert_eq!(total_bindings, 0);
12895 assert_eq!(
12896 total_bindings,
12897 roots.iter().map(|r| r.bound + r.pending).sum::<u64>() + unknown_root_bindings
12898 );
12899 }
12900
12901 fn fed_admission_facts_vectors() -> Vec<(String, Value)> {
12908 let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
12909 .join("tests/fixtures/fed/admission-facts-emit.jsonl");
12910 let text = std::fs::read_to_string(&path)
12911 .unwrap_or_else(|err| panic!("vendored fed corpus unreadable at {path:?}: {err}"));
12912 let vectors: Vec<(String, Value)> = text
12913 .lines()
12914 .filter(|line| !line.trim().is_empty())
12915 .map(|line| {
12916 let entry: Value = serde_json::from_str(line).expect("corpus line must be JSON");
12917 let id = entry["corpus_id"]
12918 .as_str()
12919 .expect("every vector carries a corpus_id")
12920 .to_string();
12921 (id, entry["package"].clone())
12922 })
12923 .collect();
12924 assert_eq!(
12928 vectors.len(),
12929 3,
12930 "vendored fed corpus changed size; re-sync from subc-federation"
12931 );
12932
12933 const COMMONLY_MODELLED: [&str; 3] = ["schema", "verified_class", "org"];
12948 let richest = vectors
12949 .iter()
12950 .filter_map(|(_, package)| package.as_object())
12951 .map(|object| {
12952 object
12953 .keys()
12954 .filter(|key| !COMMONLY_MODELLED.contains(&key.as_str()))
12955 .count()
12956 })
12957 .max()
12958 .unwrap_or(0);
12959 assert!(
12960 richest >= 2,
12961 "vendored corpus no longer carries a package with unmodelled fields, \
12962 so the relay test can no longer distinguish a verbatim relay from a lossy one"
12963 );
12964
12965 vectors
12966 }
12967
12968 #[tokio::test]
12976 async fn admission_facts_relay_carries_vendored_packages_verbatim() {
12977 for (corpus_id, package) in fed_admission_facts_vectors() {
12978 let registry = Arc::new(Registry::default());
12979 let forwarding = Arc::new(ForwardingTable::default());
12980 let supervisor = SupervisorHandle::new();
12981 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
12982 let handler =
12983 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
12984 .with_supervisor(supervisor)
12985 .with_admission_facts_config(
12986 Some("fed".to_string()),
12987 Some(vec!["target".to_string()]),
12988 );
12989
12990 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(90));
12991 hello_via_sink(
12992 &handler,
12993 &target_ctx,
12994 &mut target_rx,
12995 hello_frame("target", PROTOCOL_VERSION, 1),
12996 )
12997 .await;
12998
12999 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(91));
13000 let route_handler = handler.clone();
13001 let expected = package.clone();
13002 let route_task = tokio::spawn(async move {
13003 route_handler
13004 .handle_control_frame(
13005 &client_ctx,
13006 route_open_frame_with_admission_facts(
13007 20,
13008 "target",
13009 unique_project_root("admission-facts"),
13010 Some(subc_control::ConsumerIdentity {
13011 module_id: "fed".to_string(),
13012 launch_nonce: "fed-nonce".to_string(),
13013 }),
13014 Some(package),
13015 ),
13016 )
13017 .await
13018 .unwrap()
13019 });
13020
13021 let bind_frame = target_rx.recv().await.unwrap();
13022 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13023 let ModuleControlRequest::RouteBind {
13024 admission_facts, ..
13025 } = bind
13026 else {
13027 panic!("{corpus_id}: expected route.bind")
13028 };
13029 assert_eq!(
13030 admission_facts,
13031 Some(expected),
13032 "{corpus_id}: relay must not add, drop or reshape any field"
13033 );
13034
13035 handler
13036 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13037 .await
13038 .unwrap();
13039 route_task.await.unwrap();
13040 }
13041 }
13042
13043 #[tokio::test]
13044 async fn route_bind_never_relays_the_openers_build_class() {
13045 let registry = Arc::new(Registry::default());
13046 let forwarding = Arc::new(ForwardingTable::default());
13047 let handler =
13048 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13049 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(80));
13050 hello_via_sink(
13051 &handler,
13052 &target_ctx,
13053 &mut target_rx,
13054 hello_frame("target", PROTOCOL_VERSION, 1),
13055 )
13056 .await;
13057
13058 let mut identity = BindIdentity::new("unit".to_string(), "s".to_string());
13059 identity.build_class = Some(subc_protocol::BUILD_CLASS_PRODUCTION.to_string());
13060 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
13061 busy_refusals: None,
13062 target: RouteTarget::ToolProvider {
13063 module_id: "target".to_string(),
13064 },
13065 identity,
13066 consumer_identity: None,
13067 consumer_capabilities: None,
13068 role_versions: None,
13069 admission_facts: None,
13070 scope: None,
13071 })
13072 .unwrap();
13073 let open = Frame::build(FrameType::Request, control_flags(), 0, 0, 30, body).unwrap();
13074
13075 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(81));
13076 let route_handler = handler.clone();
13077 let route_task = tokio::spawn(async move {
13078 route_handler
13079 .handle_control_frame(&client_ctx, open)
13080 .await
13081 .unwrap()
13082 });
13083 let bind_frame = target_rx.recv().await.unwrap();
13084 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13085 let ModuleControlRequest::RouteBind { identity, .. } = bind else {
13086 panic!("expected route.bind")
13087 };
13088 assert_eq!(
13089 identity.build_class, None,
13090 "route.bind carried the build class"
13091 );
13092 assert!(
13093 !String::from_utf8_lossy(&bind_frame.body).contains("build_class"),
13094 "the route.bind bytes still name build_class"
13095 );
13096 handler
13097 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13098 .await
13099 .unwrap();
13100 route_task.await.unwrap();
13101 }
13102
13103 #[tokio::test]
13104 async fn admission_facts_gate_checks_carrier_target_and_precedence() {
13105 let registry = Arc::new(Registry::default());
13106 let forwarding = Arc::new(ForwardingTable::default());
13107 let supervisor = SupervisorHandle::new();
13108 supervisor.set_spawn_nonce("fed", "fed-nonce".to_string());
13109 supervisor.set_spawn_nonce("other", "other-nonce".to_string());
13110 let handler =
13111 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13112 .with_supervisor(supervisor)
13113 .with_admission_facts_config(
13114 Some("fed".to_string()),
13115 Some(vec!["target".to_string()]),
13116 );
13117
13118 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(70));
13119 hello_via_sink(
13120 &handler,
13121 &target_ctx,
13122 &mut target_rx,
13123 hello_frame("target", PROTOCOL_VERSION, 1),
13124 )
13125 .await;
13126 let (other_ctx, mut other_rx) = route_ctx(ConnectionId::new(71));
13127 hello_via_sink(
13128 &handler,
13129 &other_ctx,
13130 &mut other_rx,
13131 hello_frame("other", PROTOCOL_VERSION, 2),
13132 )
13133 .await;
13134
13135 let facts = json!({"schema": 1, "verified_class": "member", "org": "01H"});
13136 let expected_facts = facts.clone();
13137 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(72));
13138 let route_handler = handler.clone();
13139 let route_task = tokio::spawn(async move {
13140 route_handler
13141 .handle_control_frame(
13142 &client_ctx,
13143 route_open_frame_with_admission_facts(
13144 10,
13145 "target",
13146 unique_project_root("admission-facts"),
13147 Some(subc_control::ConsumerIdentity {
13148 module_id: "fed".to_string(),
13149 launch_nonce: "fed-nonce".to_string(),
13150 }),
13151 Some(facts.clone()),
13152 ),
13153 )
13154 .await
13155 .unwrap()
13156 });
13157 let bind_frame = target_rx.recv().await.unwrap();
13158 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13159 let ModuleControlRequest::RouteBind {
13160 admission_facts, ..
13161 } = bind
13162 else {
13163 panic!("expected route.bind")
13164 };
13165 assert_eq!(admission_facts, Some(expected_facts));
13166 handler
13167 .handle_control_frame(&target_ctx, route_bind_ack(bind_frame.header.corr))
13168 .await
13169 .unwrap();
13170 assert!(route_task.await.unwrap().is_empty());
13171 assert!(matches!(
13172 serde_json::from_slice::<ClientControlResponse>(&client_rx.recv().await.unwrap().body)
13173 .unwrap(),
13174 ClientControlResponse::RouteOpen { .. }
13175 ));
13176
13177 let direct = handler
13178 .handle_control_frame(
13179 &route_ctx(ConnectionId::new(73)).0,
13180 route_open_frame_with_admission_facts(
13181 11,
13182 "target",
13183 unique_project_root("admission-facts"),
13184 None,
13185 Some(json!({"x": 1})),
13186 ),
13187 )
13188 .await
13189 .unwrap();
13190 assert_eq!(
13191 parse_error(&direct[0])["code"],
13192 "admission_facts_not_permitted"
13193 );
13194
13195 let different_reserved = handler
13196 .handle_control_frame(
13197 &route_ctx(ConnectionId::new(77)).0,
13198 route_open_frame_with_admission_facts(
13199 15,
13200 "target",
13201 unique_project_root("admission-facts"),
13202 Some(subc_control::ConsumerIdentity {
13203 module_id: "other".to_string(),
13204 launch_nonce: "other-nonce".to_string(),
13205 }),
13206 Some(json!({"x": 1})),
13207 ),
13208 )
13209 .await
13210 .unwrap();
13211 assert_eq!(
13212 parse_error(&different_reserved[0])["code"],
13213 "admission_facts_not_permitted"
13214 );
13215
13216 let other_target = handler
13217 .handle_control_frame(
13218 &route_ctx(ConnectionId::new(74)).0,
13219 route_open_frame_with_admission_facts(
13220 12,
13221 "other",
13222 unique_project_root("admission-facts"),
13223 Some(subc_control::ConsumerIdentity {
13224 module_id: "fed".to_string(),
13225 launch_nonce: "fed-nonce".to_string(),
13226 }),
13227 Some(json!({"x": 1})),
13228 ),
13229 )
13230 .await
13231 .unwrap();
13232 assert_eq!(
13233 parse_error(&other_target[0])["code"],
13234 "admission_facts_target_not_allowed"
13235 );
13236
13237 let nonexistent = handler
13238 .handle_control_frame(
13239 &route_ctx(ConnectionId::new(75)).0,
13240 route_open_frame_with_admission_facts(
13241 13,
13242 "missing",
13243 unique_project_root("admission-facts"),
13244 None,
13245 Some(json!({"x": 1})),
13246 ),
13247 )
13248 .await
13249 .unwrap();
13250 assert_eq!(parse_error(&nonexistent[0])["code"], "unknown_module");
13251
13252 let described = handler
13253 .handle_control_frame(
13254 &route_ctx(ConnectionId::new(76)).0,
13255 Frame::build(
13256 FrameType::Request,
13257 control_flags(),
13258 0,
13259 0,
13260 14,
13261 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13262 )
13263 .unwrap(),
13264 )
13265 .await
13266 .unwrap();
13267 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13268 serde_json::from_slice(&described[0].body).unwrap()
13269 else {
13270 panic!("expected server.describe response")
13271 };
13272 assert!(capabilities
13273 .iter()
13274 .any(|cap| cap == "admission_facts_relay_v1"));
13275 }
13276
13277 #[tokio::test]
13278 async fn admission_facts_without_configured_carrier_are_rejected() {
13279 let registry = Arc::new(Registry::default());
13280 let forwarding = Arc::new(ForwardingTable::default());
13281 let handler = ControlHandler::with_forwarding(registry, forwarding);
13282 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(78));
13283 hello_via_sink(
13284 &handler,
13285 &target_ctx,
13286 &mut target_rx,
13287 hello_frame("target", PROTOCOL_VERSION, 1),
13288 )
13289 .await;
13290
13291 let responses = handler
13292 .handle_control_frame(
13293 &route_ctx(ConnectionId::new(79)).0,
13294 route_open_frame_with_admission_facts(
13295 16,
13296 "target",
13297 unique_project_root("admission-facts"),
13298 None,
13299 Some(json!({"x": 1})),
13300 ),
13301 )
13302 .await
13303 .unwrap();
13304 assert_eq!(
13305 parse_error(&responses[0])["code"],
13306 "admission_facts_not_permitted"
13307 );
13308 }
13309
13310 #[tokio::test]
13311 async fn route_open_relays_consumer_capabilities_verbatim() {
13312 let registry = Arc::new(Registry::default());
13313 let forwarding = Arc::new(ForwardingTable::default());
13314 let handler =
13315 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13316 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(37));
13317 hello_via_sink(
13318 &handler,
13319 &module_ctx,
13320 &mut module_rx,
13321 hello_frame("aft", PROTOCOL_VERSION, 7),
13322 )
13323 .await;
13324
13325 let expected = vec!["elicitation".to_string(), "roots".to_string()];
13326 let expected_for_request = expected.clone();
13327 let project_root = unique_project_root("consumer-capabilities-present");
13328 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(38));
13329 let route_handler = handler.clone();
13330 let route_task = tokio::spawn(async move {
13331 route_handler
13332 .handle_control_frame(
13333 &client_ctx,
13334 route_open_frame_with_consumer_capabilities(
13335 401,
13336 "aft",
13337 project_root,
13338 Some(expected_for_request),
13339 ),
13340 )
13341 .await
13342 .unwrap()
13343 });
13344 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13345 .await
13346 .unwrap()
13347 .unwrap();
13348 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13349 let ModuleControlRequest::RouteBind {
13350 consumer_capabilities,
13351 ..
13352 } = bind
13353 else {
13354 panic!("expected route.bind request, got {bind:?}");
13355 };
13356 assert_eq!(consumer_capabilities, Some(expected.clone()));
13357
13358 handler
13359 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13360 .await
13361 .unwrap();
13362 let route_response = route_task.await.unwrap();
13363 assert!(route_response.is_empty());
13364 let published = client_rx.recv().await.unwrap();
13365 assert!(matches!(
13366 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13367 ClientControlResponse::RouteOpen { .. }
13368 ));
13369 }
13370
13371 #[tokio::test]
13372 async fn route_open_without_consumer_capabilities_relays_none() {
13373 let registry = Arc::new(Registry::default());
13374 let forwarding = Arc::new(ForwardingTable::default());
13375 let handler =
13376 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13377 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(39));
13378 hello_via_sink(
13379 &handler,
13380 &module_ctx,
13381 &mut module_rx,
13382 hello_frame("aft", PROTOCOL_VERSION, 7),
13383 )
13384 .await;
13385
13386 let project_root = unique_project_root("consumer-capabilities-absent");
13387 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(40));
13388 let route_handler = handler.clone();
13389 let route_task = tokio::spawn(async move {
13390 route_handler
13391 .handle_control_frame(&client_ctx, route_open_frame(402, "aft", project_root))
13392 .await
13393 .unwrap()
13394 });
13395 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13396 .await
13397 .unwrap()
13398 .unwrap();
13399 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13400 let ModuleControlRequest::RouteBind {
13401 consumer_capabilities,
13402 ..
13403 } = bind
13404 else {
13405 panic!("expected route.bind request, got {bind:?}");
13406 };
13407 assert_eq!(consumer_capabilities, None);
13408
13409 handler
13410 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13411 .await
13412 .unwrap();
13413 let route_response = route_task.await.unwrap();
13414 assert!(route_response.is_empty());
13415 let published = client_rx.recv().await.unwrap();
13416 assert!(matches!(
13417 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13418 ClientControlResponse::RouteOpen { .. }
13419 ));
13420 }
13421
13422 async fn bind_role_versions_for(
13426 sent: Option<BTreeMap<String, String>>,
13427 connection: u64,
13428 ) -> Option<BTreeMap<String, String>> {
13429 let registry = Arc::new(Registry::default());
13430 let forwarding = Arc::new(ForwardingTable::default());
13431 let handler =
13432 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13433 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(connection));
13434 hello_via_sink(
13435 &handler,
13436 &module_ctx,
13437 &mut module_rx,
13438 hello_frame("aft", PROTOCOL_VERSION, 7),
13439 )
13440 .await;
13441 let project_root = unique_project_root("role-versions");
13442 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(connection + 1));
13443 let route_handler = handler.clone();
13444 let route_task = tokio::spawn(async move {
13445 route_handler
13446 .handle_control_frame(
13447 &client_ctx,
13448 route_open_frame_with_role_versions(403, "aft", project_root, sent),
13449 )
13450 .await
13451 .unwrap()
13452 });
13453 let bind_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13454 .await
13455 .expect("a well-formed route.open reaches the module as a bind")
13456 .unwrap();
13457 let bind: ModuleControlRequest = serde_json::from_slice(&bind_frame.body).unwrap();
13458 let ModuleControlRequest::RouteBind { role_versions, .. } = bind else {
13459 panic!("expected route.bind request, got {bind:?}");
13460 };
13461 handler
13462 .handle_control_frame(&module_ctx, route_bind_ack(bind_frame.header.corr))
13463 .await
13464 .unwrap();
13465 assert!(route_task.await.unwrap().is_empty());
13466 let published = client_rx.recv().await.unwrap();
13467 assert!(matches!(
13468 serde_json::from_slice::<ClientControlResponse>(&published.body).unwrap(),
13469 ClientControlResponse::RouteOpen { .. }
13470 ));
13471 role_versions
13472 }
13473
13474 #[tokio::test]
13475 async fn route_open_relays_role_versions_verbatim() {
13476 let sent = role_versions(&[("tool-provider", "v1"), ("management-surface", "v12")]);
13477 assert_eq!(
13478 bind_role_versions_for(Some(sent.clone()), 141).await,
13479 Some(sent)
13480 );
13481 }
13482
13483 #[tokio::test]
13486 async fn route_open_with_empty_or_absent_role_versions_relays_none() {
13487 assert_eq!(bind_role_versions_for(None, 143).await, None);
13488 assert_eq!(
13489 bind_role_versions_for(Some(BTreeMap::new()), 145).await,
13490 None
13491 );
13492 }
13493
13494 #[tokio::test]
13495 async fn route_open_refuses_malformed_role_versions_before_any_bind() {
13496 let registry = Arc::new(Registry::default());
13497 let forwarding = Arc::new(ForwardingTable::default());
13498 let handler =
13499 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
13500 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(147));
13501 hello_via_sink(
13502 &handler,
13503 &module_ctx,
13504 &mut module_rx,
13505 hello_frame("aft", PROTOCOL_VERSION, 7),
13506 )
13507 .await;
13508
13509 let nine: BTreeMap<String, String> = (0..9)
13510 .map(|index| (format!("role-{index}"), "v1".to_string()))
13511 .collect();
13512 for (label, malformed) in [
13513 (
13514 "invalid role name",
13515 role_versions(&[("Tool_Provider", "v1")]),
13516 ),
13517 ("invalid version", role_versions(&[("tool-provider", "v0")])),
13518 ("nine entries", nine),
13519 ] {
13520 let responses = tokio::time::timeout(
13523 Duration::from_secs(1),
13524 handler.handle_control_frame(
13525 &route_ctx(ConnectionId::new(148)).0,
13526 route_open_frame_with_role_versions(
13527 404,
13528 "aft",
13529 unique_project_root("role-versions-malformed"),
13530 Some(malformed),
13531 ),
13532 ),
13533 )
13534 .await
13535 .unwrap_or_else(|_| panic!("{label}: the open was relayed instead of refused"))
13536 .unwrap();
13537 assert_eq!(responses.len(), 1, "{label}");
13538 assert_eq!(responses[0].header.ty, FrameType::Error, "{label}");
13539 let error = parse_error(&responses[0]);
13540 assert_eq!(error["code"], "invalid_request", "{label}: {error}");
13541 assert_eq!(
13542 error["detail"]["field"], "role_versions",
13543 "{label}: {error}"
13544 );
13545 assert!(
13546 !error_codes::is_retryable_route_open(error["code"].as_str().unwrap()),
13547 "{label}: a malformed declaration is terminal"
13548 );
13549 assert!(
13550 module_rx.try_recv().is_err(),
13551 "{label}: the module must never see a bind"
13552 );
13553 }
13554 }
13555
13556 #[tokio::test]
13560 async fn route_role_versions_capability_is_advertised() {
13561 let handler = ControlHandler::new(Arc::new(Registry::default()));
13562 let (ctx, mut rx) = route_ctx(ConnectionId::new(149));
13563 let ack = hello_via_sink(
13564 &handler,
13565 &ctx,
13566 &mut rx,
13567 hello_frame("m", PROTOCOL_VERSION, 1),
13568 )
13569 .await;
13570 let ack = parse_ack(&ack);
13571 assert!(
13572 ack.subc_capabilities
13573 .iter()
13574 .any(|c| c == "route-role-versions/v1"),
13575 "{:?}",
13576 ack.subc_capabilities
13577 );
13578
13579 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13580 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13581 let reply = handler
13582 .handle_control_frame(&route_ctx(ConnectionId::new(150)).0, frame)
13583 .await
13584 .unwrap()
13585 .pop()
13586 .unwrap();
13587 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13588 serde_json::from_slice(&reply.body).unwrap()
13589 else {
13590 panic!("not a server.describe reply");
13591 };
13592 assert!(
13593 capabilities.iter().any(|c| c == CAP_ROUTE_ROLE_VERSIONS_V1),
13594 "{capabilities:?}"
13595 );
13596 }
13597
13598 #[tokio::test]
13602 async fn catalog_events_capability_is_advertised() {
13603 let handler = ControlHandler::new(Arc::new(Registry::default()));
13604 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13605 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13606 let reply = handler
13607 .handle_control_frame(&route_ctx(ConnectionId::new(151)).0, frame)
13608 .await
13609 .unwrap()
13610 .pop()
13611 .unwrap();
13612 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13613 serde_json::from_slice(&reply.body).unwrap()
13614 else {
13615 panic!("not a server.describe reply");
13616 };
13617 assert!(
13618 capabilities
13619 .iter()
13620 .any(|c| c == subc_protocol::manifest::CAP_CATALOG_EVENTS_V1),
13621 "{capabilities:?}"
13622 );
13623 }
13624
13625 #[tokio::test]
13626 async fn scope_bind_proof_capability_is_advertised_in_hello_ack_and_server_describe() {
13627 let handler = ControlHandler::new(Arc::new(Registry::default()));
13628 let (module, mut module_rx) = route_ctx(ConnectionId::new(152));
13629 let ack = hello_via_sink(
13630 &handler,
13631 &module,
13632 &mut module_rx,
13633 hello_frame("scope-proof-owner", PROTOCOL_VERSION, 1),
13634 )
13635 .await;
13636 let ack = parse_ack(&ack);
13637 assert!(
13638 ack.subc_capabilities
13639 .iter()
13640 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13641 "{:?}",
13642 ack.subc_capabilities
13643 );
13644 assert!(ack
13645 .subc_capabilities
13646 .iter()
13647 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13648
13649 let (client, _client_rx) = route_ctx(ConnectionId::new(153));
13650 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13651 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13652 let reply = handler
13653 .handle_control_frame(&client, frame)
13654 .await
13655 .unwrap()
13656 .pop()
13657 .unwrap();
13658 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13659 serde_json::from_slice(&reply.body).unwrap()
13660 else {
13661 panic!("not a server.describe reply");
13662 };
13663 assert!(
13664 capabilities
13665 .iter()
13666 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V1),
13667 "{capabilities:?}"
13668 );
13669 assert!(capabilities
13670 .iter()
13671 .any(|c| c == subc_protocol::manifest::CAP_SCOPE_BIND_PROOF_V2));
13672 }
13673
13674 #[tokio::test]
13675 async fn scope_on_behalf_of_capability_is_advertised_in_hello_ack_and_server_describe() {
13676 let handler = ControlHandler::new(Arc::new(Registry::default()));
13677 let (module, mut rx) = route_ctx(ConnectionId::new(156));
13678 let ack = hello_via_sink(
13679 &handler,
13680 &module,
13681 &mut rx,
13682 hello_frame("attribution-owner", PROTOCOL_VERSION, 1),
13683 )
13684 .await;
13685 assert!(parse_ack(&ack)
13686 .subc_capabilities
13687 .iter()
13688 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13689 assert!(parse_ack(&ack)
13690 .subc_capabilities
13691 .iter()
13692 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13693 let (client, _rx) = route_ctx(ConnectionId::new(157));
13694 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13695 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13696 let reply = handler
13697 .handle_control_frame(&client, frame)
13698 .await
13699 .unwrap()
13700 .pop()
13701 .unwrap();
13702 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13703 serde_json::from_slice(&reply.body).unwrap()
13704 else {
13705 panic!("server.describe reply")
13706 };
13707 assert!(capabilities
13708 .iter()
13709 .any(|cap| cap == subc_protocol::scope::SCOPE_ON_BEHALF_OF_CAPABILITY));
13710 assert!(capabilities
13711 .iter()
13712 .any(|cap| cap == subc_protocol::scope::SCOPE_SERVED_BY_CAPABILITY));
13713 }
13714
13715 #[tokio::test]
13716 async fn scope_workdir_capability_is_advertised_in_hello_ack_and_server_describe() {
13717 let handler = ControlHandler::new(Arc::new(Registry::default()));
13718 let (module, mut rx) = route_ctx(ConnectionId::new(158));
13719 let ack = hello_via_sink(
13720 &handler,
13721 &module,
13722 &mut rx,
13723 hello_frame("project-scope-owner", PROTOCOL_VERSION, 1),
13724 )
13725 .await;
13726 assert!(parse_ack(&ack)
13727 .subc_capabilities
13728 .iter()
13729 .any(|cap| cap == "scope-workdir/v1"));
13730 let (client, _rx) = route_ctx(ConnectionId::new(159));
13731 let frame = Frame::build(
13732 FrameType::Request,
13733 control_flags(),
13734 0,
13735 0,
13736 5,
13737 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13738 )
13739 .unwrap();
13740 let reply = handler
13741 .handle_control_frame(&client, frame)
13742 .await
13743 .unwrap()
13744 .remove(0);
13745 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13746 serde_json::from_slice(&reply.body).unwrap()
13747 else {
13748 panic!("server.describe reply")
13749 };
13750 assert!(capabilities.iter().any(|cap| cap == "scope-workdir/v1"));
13751 }
13752
13753 #[tokio::test]
13754 async fn worker_runs_capability_is_advertised_in_hello_ack_and_server_describe() {
13755 let handler = ControlHandler::new(Arc::new(Registry::default()));
13756 let (module, mut rx) = route_ctx(ConnectionId::new(160));
13757 let ack = hello_via_sink(
13758 &handler,
13759 &module,
13760 &mut rx,
13761 hello_frame("worker-run-owner", PROTOCOL_VERSION, 1),
13762 )
13763 .await;
13764 assert!(parse_ack(&ack)
13765 .subc_capabilities
13766 .iter()
13767 .any(|cap| cap == "worker-runs/v1"));
13768 let (client, _rx) = route_ctx(ConnectionId::new(161));
13769 let frame = Frame::build(
13770 FrameType::Request,
13771 control_flags(),
13772 0,
13773 0,
13774 5,
13775 serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap(),
13776 )
13777 .unwrap();
13778 let reply = handler
13779 .handle_control_frame(&client, frame)
13780 .await
13781 .unwrap()
13782 .remove(0);
13783 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13784 serde_json::from_slice(&reply.body).unwrap()
13785 else {
13786 panic!("server.describe reply")
13787 };
13788 assert!(capabilities.iter().any(|cap| cap == "worker-runs/v1"));
13789 }
13790
13791 #[tokio::test]
13792 async fn scope_replace_if_idle_capability_is_advertised_in_hello_ack_and_server_describe() {
13793 let handler = ControlHandler::new(Arc::new(Registry::default()));
13794 let (module, mut rx) = route_ctx(ConnectionId::new(154));
13795 let ack = hello_via_sink(
13796 &handler,
13797 &module,
13798 &mut rx,
13799 hello_frame("idle-scope-owner", PROTOCOL_VERSION, 1),
13800 )
13801 .await;
13802 assert!(parse_ack(&ack)
13803 .subc_capabilities
13804 .iter()
13805 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13806 let (client, _rx) = route_ctx(ConnectionId::new(155));
13807 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
13808 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
13809 let reply = handler
13810 .handle_control_frame(&client, frame)
13811 .await
13812 .unwrap()
13813 .pop()
13814 .unwrap();
13815 let ClientControlResponse::ServerDescribe { capabilities, .. } =
13816 serde_json::from_slice(&reply.body).unwrap()
13817 else {
13818 panic!("not a server.describe reply")
13819 };
13820 assert!(capabilities
13821 .iter()
13822 .any(|cap| cap == subc_protocol::manifest::CAP_SCOPE_REPLACE_IF_IDLE_V1));
13823 }
13824
13825 #[tokio::test]
13826 async fn supervision_only_module_health_probe_does_not_enable_route_open_and_cleans_up() {
13827 let registry = Arc::new(Registry::default());
13828 let forwarding = Arc::new(ForwardingTable::default());
13829 let handler =
13830 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
13831 .with_health_probe_timeout(Duration::from_secs(5));
13832 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(35));
13833 hello_via_sink(
13834 &handler,
13835 &module_ctx,
13836 &mut module_rx,
13837 non_routable_hello_frame_with_control_ops(
13838 "mcp",
13839 300,
13840 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
13841 ),
13842 )
13843 .await;
13844 assert!(registry
13845 .get_module("mcp")
13846 .unwrap()
13847 .unwrap()
13848 .manifest
13849 .provides
13850 .is_empty());
13851
13852 let (route_client_ctx, _route_client_rx) = route_ctx(ConnectionId::new(36));
13853 let route_response = handler
13854 .handle_control_frame(
13855 &route_client_ctx,
13856 route_open_frame(301, "mcp", unique_project_root("non-routable-mcp")),
13857 )
13858 .await
13859 .unwrap();
13860 assert_eq!(route_response[0].header.ty, FrameType::Error);
13861 assert_eq!(
13862 parse_error(&route_response[0])["code"],
13863 "target_unavailable"
13864 );
13865 assert!(parse_error(&route_response[0])["message"]
13866 .as_str()
13867 .unwrap()
13868 .contains("does not provide the requested target"));
13869 assert!(module_rx.try_recv().is_err());
13870
13871 let (health_client_ctx, _health_client_rx) = route_ctx(ConnectionId::new(37));
13872 let health_handler = handler.clone();
13873 let health_task = tokio::spawn(async move {
13874 health_handler
13875 .handle_control_frame(
13876 &health_client_ctx,
13877 supervisor_health_probe_frame(302, "mcp"),
13878 )
13879 .await
13880 .unwrap()
13881 });
13882 let health_frame = tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
13883 .await
13884 .unwrap()
13885 .unwrap();
13886 assert_eq!(
13887 serde_json::from_slice::<ModuleControlRequest>(&health_frame.body).unwrap(),
13888 ModuleControlRequest::HealthCheck {}
13889 );
13890 handler
13891 .handle_control_frame(
13892 &module_ctx,
13893 health_response(health_frame.header.corr, HealthStatus::Ok),
13894 )
13895 .await
13896 .unwrap();
13897 let health_response = health_task.await.unwrap();
13898 assert_eq!(health_response[0].header.ty, FrameType::Response);
13899 match serde_json::from_slice::<ClientControlResponse>(&health_response[0].body).unwrap() {
13900 ClientControlResponse::SupervisorHealthProbe {
13901 module_id, status, ..
13902 } => {
13903 assert_eq!(module_id, "mcp");
13904 assert_eq!(status, HealthStatus::Ok);
13905 }
13906 other => panic!("unexpected health response: {other:?}"),
13907 }
13908
13909 forwarding
13914 .cleanup_connection(module_ctx.connection_id)
13915 .unwrap();
13916 let (cleanup_probe_ctx, _cleanup_probe_rx) = route_ctx(ConnectionId::new(38));
13917 let cleanup_response = tokio::time::timeout(
13918 Duration::from_millis(200),
13919 handler.handle_control_frame(
13920 &cleanup_probe_ctx,
13921 supervisor_health_probe_frame(303, "mcp"),
13922 ),
13923 )
13924 .await
13925 .expect("probe should fail immediately when the control lane is gone")
13926 .unwrap();
13927 assert_eq!(cleanup_response[0].header.ty, FrameType::Error);
13928 assert_eq!(
13929 parse_error(&cleanup_response[0])["code"],
13930 "target_unavailable"
13931 );
13932 assert!(parse_error(&cleanup_response[0])["message"]
13933 .as_str()
13934 .unwrap()
13935 .contains("no module connection"));
13936
13937 handler
13938 .cleanup_connection(module_ctx.connection_id)
13939 .unwrap();
13940 }
13941
13942 #[tokio::test]
13943 async fn route_open_classifies_unregistered_running_supervised_module_as_warming() {
13944 let registry = Arc::new(Registry::default());
13945 let supervisor_handle = SupervisorHandle::new();
13946 let supervisor =
13947 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
13948 .with_handle(supervisor_handle.clone())
13949 .with_connection_file_path(
13950 std::env::temp_dir()
13951 .join(format!("subc-route-open-warming-{}", std::process::id())),
13952 );
13953 let module = supervisor
13954 .supervise_configured(
13955 ModuleSpec {
13956 carry_containment: false,
13957 module_id: "warming".to_string(),
13958 program: fake_aft_stub_path(),
13959 args: Vec::new(),
13960 env: Vec::new(),
13961 reserved: false,
13962 reserved_prefixes: Vec::new(),
13963 protocol: ModuleProtocol::Subc,
13964 overlap: Default::default(),
13965 },
13966 true,
13967 )
13968 .unwrap();
13969 assert_eq!(module.state().unwrap(), ModuleState::Running);
13970
13971 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
13972 let (ctx, _rx) = route_ctx(ConnectionId::new(39));
13973 let response = handler
13974 .handle_control_frame(
13975 &ctx,
13976 route_open_frame(304, "warming", unique_project_root("warming")),
13977 )
13978 .await
13979 .unwrap();
13980 module.stop().await.unwrap();
13981
13982 assert_eq!(response[0].header.ty, FrameType::Error);
13983 let error = parse_error(&response[0]);
13984 assert_eq!(error["code"], "module_warming");
13985 assert!(error["message"]
13986 .as_str()
13987 .unwrap()
13988 .contains("state=running, enabled=true, live=false"));
13989 }
13990
13991 #[test]
13992 fn route_open_connection_cap_logs_admission_reason_and_capacity() {
13993 let handler = ControlHandler::new(Arc::new(Registry::default()));
13994 let capture = EventCapture::default();
13995 let _subscriber =
13996 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
13997 let (ctx, _rx) = route_ctx(ConnectionId::new(96));
13998 let limit = crate::server::MAX_PENDING_ROUTE_OPENS_PER_CONNECTION;
13999 let pending = (0..limit).collect::<Vec<_>>();
14000 let response = handler
14001 .route_open_capacity_refusal(
14002 &ctx,
14003 &route_open_frame(396, "busy", unique_project_root("connection-cap")),
14004 "busy",
14005 pending.len(),
14006 limit,
14007 )
14008 .unwrap();
14009 assert_eq!(parse_error(&response)["code"], "target_unavailable");
14010 let event = capture
14011 .events()
14012 .into_iter()
14013 .find(|event| {
14014 event.target == "control"
14015 && event.fields.get("reason") == Some(&"\"open_admission_full\"".to_string())
14016 })
14017 .expect("connection admission refusal event");
14018 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
14019 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
14020 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14023 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14024 }
14025
14026 #[test]
14027 fn route_open_target_cap_logs_admission_reason_and_capacity() {
14028 let handler = ControlHandler::new(Arc::new(Registry::default()));
14029 let capture = EventCapture::default();
14030 let _subscriber =
14031 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14032 let (ctx, _rx) = route_ctx(ConnectionId::new(97));
14033 let limit = MAX_PENDING_ROUTE_BINDS_PER_TARGET;
14034 let guards = (0..limit)
14035 .map(|_| {
14036 handler
14037 .route_bind_concurrency
14038 .try_admit("busy", limit)
14039 .unwrap()
14040 })
14041 .collect::<Vec<_>>();
14042 let in_flight = match handler.route_bind_concurrency.try_admit("busy", limit) {
14043 Err(in_flight) => in_flight,
14044 Ok(_) => panic!("target cap must refuse after {limit} admissions"),
14045 };
14046 let response = handler
14047 .route_open_target_capacity_refusal(
14048 &ctx,
14049 &RouteOpenCaller::unknown(),
14050 &route_open_frame(397, "busy", unique_project_root("target-cap")),
14051 "busy",
14052 in_flight,
14053 )
14054 .unwrap();
14055 assert_eq!(parse_error(&response)["code"], "target_unavailable");
14056 let event = capture
14057 .events()
14058 .into_iter()
14059 .find(|event| {
14060 event.target == "control"
14061 && event.fields.get("reason") == Some(&"\"target_binds_full\"".to_string())
14062 })
14063 .expect("target admission refusal event");
14064 assert_eq!(event.fields.get("in_flight"), Some(&limit.to_string()));
14065 assert_eq!(event.fields.get("limit"), Some(&limit.to_string()));
14066 drop(guards);
14067 }
14068
14069 #[cfg(unix)]
14070 #[tokio::test(start_paused = true)]
14071 async fn supervisor_list_shows_a_failed_modules_next_retry() {
14072 let registry = Arc::new(Registry::default());
14073 let handle = SupervisorHandle::new();
14074 let supervisor =
14075 Supervisor::new_for_test(registry.clone(), RestartPolicy::new(0, Duration::ZERO))
14076 .with_handle(handle.clone());
14077 let module = supervisor
14078 .spawn(crate::ModuleSpec {
14079 carry_containment: false,
14080 module_id: "retry-list".into(),
14081 program: PathBuf::from("/bin/sh"),
14082 args: vec!["-c".into(), "exit 17".into()],
14083 env: vec![],
14084 reserved: false,
14085 reserved_prefixes: vec![],
14086 protocol: ModuleProtocol::None,
14087 overlap: Default::default(),
14088 })
14089 .unwrap();
14090 let deadline = std::time::Instant::now() + Duration::from_secs(10);
14091 while module.state().unwrap() != crate::ModuleState::Failed {
14092 assert!(
14093 std::time::Instant::now() < deadline,
14094 "waiting for budget failure"
14095 );
14096 tokio::task::yield_now().await;
14097 }
14098 let handler = ControlHandler::new(registry).with_supervisor(handle);
14099 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14100 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14101 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14102 assert_eq!(body["modules"][0]["state"], "failed");
14103 let retry = body["modules"][0]["next_retry_at_ms"]
14104 .as_u64()
14105 .expect("scheduled retry on the wire");
14106 let now = std::time::SystemTime::now()
14107 .duration_since(std::time::UNIX_EPOCH)
14108 .unwrap()
14109 .as_millis() as u64;
14110 assert!((599_000..=600_000).contains(&retry.saturating_sub(now)));
14111 module.set_enabled(false).await.unwrap();
14112 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14113 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14114 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14115 assert!(body["modules"][0].get("next_retry_at_ms").is_none());
14116 }
14117
14118 #[tokio::test]
14119 async fn slow_step_before_pid_read_does_not_spend_its_budget() {
14120 let _guard = crate::subtree_resources::fault_read_guard().await;
14121 let dir = cortexkit_test_support::ScratchDir::new("pid-read-budget");
14122 let supervisor =
14123 Supervisor::new_for_test(Arc::new(Registry::default()), RestartPolicy::default());
14124 let module = supervisor
14125 .spawn(crate::ModuleSpec {
14126 carry_containment: false,
14127 module_id: "pid-read-budget".into(),
14128 program: std::env::current_exe().unwrap(),
14129 args: vec![
14130 "--exact".into(),
14131 "control::tests::pid_resource_read_fixture".into(),
14132 ],
14133 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14134 .into_iter()
14135 .map(|key| (key.into(), dir.display().to_string()))
14136 .chain(std::iter::once((
14137 "SUBC_SUBTREE_PID_FIXTURE".into(),
14138 "1".into(),
14139 )))
14140 .collect(),
14141 reserved: false,
14142 reserved_prefixes: Vec::new(),
14143 protocol: ModuleProtocol::None,
14144 overlap: Default::default(),
14145 })
14146 .unwrap();
14147 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
14148 while !matches!(
14149 module.child_resource_usage(),
14150 subc_control::ChildResourceUsage::Measured(_)
14151 ) {
14152 assert!(
14153 tokio::time::Instant::now() < ready_deadline,
14154 "fixture process was not measurable"
14155 );
14156 tokio::time::sleep(Duration::from_millis(10)).await;
14157 }
14158 tokio::time::sleep(crate::subtree_resources::READ_TIMEOUT * 2).await;
14159 let reading = crate::subtree_resources::read_process(module.clone()).await;
14160 module.stop().await.unwrap();
14161 assert!(
14162 matches!(reading, subc_control::ChildResourceUsage::Measured(_)),
14163 "{reading:?}"
14164 );
14165 }
14166
14167 #[test]
14168 fn pid_resource_read_fixture() {
14169 if std::env::var_os("SUBC_SUBTREE_PID_FIXTURE").is_some() {
14170 loop {
14171 std::thread::park_timeout(Duration::from_secs(600));
14172 }
14173 }
14174 }
14175
14176 #[tokio::test]
14177 async fn disabled_module_tree_is_not_running_instead_of_permanently_pending() {
14178 let registry = Arc::new(Registry::default());
14179 let handle = SupervisorHandle::new();
14180 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14181 .with_handle(handle.clone());
14182 supervisor
14183 .supervise_configured(
14184 crate::ModuleSpec {
14185 module_id: "disabled-resource-reading".into(),
14186 program: std::env::current_exe().unwrap(),
14187 args: Vec::new(),
14188 env: Vec::new(),
14189 reserved: false,
14190 reserved_prefixes: Vec::new(),
14191 protocol: ModuleProtocol::None,
14192 overlap: Default::default(),
14193 carry_containment: false,
14194 },
14195 false,
14196 )
14197 .unwrap();
14198 let handler = ControlHandler::new(registry).with_supervisor(handle);
14199 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14200 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14201 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14202 assert!(body["modules"][0].get("tree").is_none());
14203 assert_eq!(body["modules"][0]["tree_unavailable"], "not_running");
14204 }
14205
14206 #[tokio::test]
14207 async fn supervisor_list_reads_process_resources_before_a_stalled_tree() {
14208 let _guard = crate::subtree_resources::fault_read_guard().await;
14209 let dir = cortexkit_test_support::ScratchDir::new("list-process-priority");
14210 let registry = Arc::new(Registry::default());
14211 let handle = SupervisorHandle::new();
14212 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14213 .with_handle(handle.clone());
14214 let module = supervisor
14215 .spawn(crate::ModuleSpec {
14216 carry_containment: false,
14217 module_id: "list-process-priority".into(),
14218 program: std::env::current_exe().unwrap(),
14219 args: vec![
14220 "--exact".into(),
14221 "control::tests::pid_resource_read_fixture".into(),
14222 ],
14223 env: ["XDG_DATA_HOME", "XDG_RUNTIME_DIR", "XDG_CONFIG_HOME"]
14224 .into_iter()
14225 .map(|key| (key.into(), dir.display().to_string()))
14226 .chain(std::iter::once((
14227 "SUBC_SUBTREE_PID_FIXTURE".into(),
14228 "1".into(),
14229 )))
14230 .collect(),
14231 reserved: false,
14232 reserved_prefixes: Vec::new(),
14233 protocol: ModuleProtocol::None,
14234 overlap: Default::default(),
14235 })
14236 .unwrap();
14237 let ready_deadline = tokio::time::Instant::now() + Duration::from_secs(5);
14238 loop {
14239 let image = module.running_image_agreement().await;
14240 if matches!(image, subc_control::RunningImageAgreement::Match { .. }) {
14241 break;
14242 }
14243 assert!(
14244 tokio::time::Instant::now() < ready_deadline,
14245 "the fixture process never became observable: {image:?}"
14246 );
14247 tokio::time::sleep(Duration::from_millis(10)).await;
14248 }
14249 let (target, release, _) = crate::subtree_resources::held();
14250 module.set_subtree_target_for_test(target);
14251 let handler = ControlHandler::new(registry).with_supervisor(handle);
14252 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14253 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14254 drop(release);
14255 module.stop().await.unwrap();
14256 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14257 assert_eq!(
14258 body["modules"][0]["resources"]["status"], "measured",
14259 "the optional tree timeout must not discard the process reading: {body}"
14260 );
14261 assert!(body["modules"][0].get("tree").is_none());
14262 assert_eq!(body["modules"][0]["tree_unavailable"], "not_yet_measured");
14263 assert_eq!(
14264 body["modules"][0]["pending_reload"]["image"]["status"], "match",
14265 "the tree timeout must not hide a completed image observation"
14266 );
14267 }
14268
14269 #[cfg(any(target_os = "linux", target_os = "macos"))]
14270 #[tokio::test]
14271 async fn cold_explicit_provenance_completes_while_list_stays_bounded() {
14272 let mut handler = ControlHandler::new(Arc::new(Registry::default()))
14273 .with_daemon_provenance(
14274 std::process::id(),
14275 0,
14276 Some(std::env::current_exe().unwrap()),
14277 None,
14278 None,
14279 );
14280 let (probe, release, began) = ExecutableIdentityProbe::held();
14281 handler.daemon_provenance.probe = probe;
14282 let observe = |handler: ControlHandler, corr| {
14283 tokio::spawn(async move {
14284 let (ctx, _) = route_ctx(ConnectionId::new(corr));
14285 let frame =
14286 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, Vec::new())
14287 .unwrap();
14288 handler
14289 .handle_supervisor_provenance(&ctx, frame, None)
14290 .await
14291 .unwrap()
14292 })
14293 };
14294 let explicit = observe(handler.clone(), 114);
14295 began.await.unwrap();
14296 let concurrent = observe(handler.clone(), 115);
14297 tokio::time::sleep(Duration::from_millis(300)).await;
14300 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14301 let list = tokio::time::timeout(
14302 Duration::from_secs(1),
14303 handler.handle_supervisor_list(frame),
14304 )
14305 .await;
14306 let explicit_was_pending = !explicit.is_finished() && !concurrent.is_finished();
14307 drop(release);
14308 let replies = explicit.await.unwrap();
14309 let concurrent_replies = concurrent.await.unwrap();
14310 assert!(
14311 explicit_was_pending,
14312 "explicit provenance must await the full observation"
14313 );
14314 let list = list
14315 .expect("a concurrent explicit probe must not stall list")
14316 .unwrap();
14317 let list_body: Value = serde_json::from_slice(&list[0].body).unwrap();
14318 assert_eq!(list_body["op"], "supervisor.list");
14319 for replies in [replies, concurrent_replies] {
14320 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14321 assert_eq!(
14322 body["daemon"]["daemon_observed"]["running_image"]["status"],
14323 "match"
14324 );
14325 }
14326 }
14327
14328 #[tokio::test]
14329 async fn supervisor_list_slow_subtree_does_not_hold_connection_reader() {
14330 let _guard = crate::subtree_resources::fault_read_guard().await;
14331 let registry = Arc::new(Registry::default());
14332 let handle = SupervisorHandle::new();
14333 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14334 .with_handle(handle.clone());
14335 let module = supervisor
14336 .supervise_configured(
14337 crate::ModuleSpec {
14338 carry_containment: false,
14339 module_id: "memory-read".into(),
14340 program: PathBuf::from("disabled-module"),
14341 args: Vec::new(),
14342 env: Vec::new(),
14343 reserved: false,
14344 reserved_prefixes: Vec::new(),
14345 protocol: ModuleProtocol::Subc,
14346 overlap: Default::default(),
14347 },
14348 true,
14349 )
14350 .unwrap();
14351 let (target, release, began) = crate::subtree_resources::held();
14352 let stalled_target = target.clone();
14355 let stalled = tokio::task::spawn_blocking(move || {
14356 stalled_target.tree(&subc_os::ProcessTable::default())
14357 });
14358 began.await.unwrap();
14359 module.set_subtree_target_for_test(target);
14360 let handler = ControlHandler::new(registry).with_supervisor(handle);
14361 let (ctx, mut rx) = route_ctx(ConnectionId::new(112));
14362 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14363 let replies = handler
14364 .handle_client_control_request(
14365 &ctx,
14366 frame,
14367 ClientControlRequest::SupervisorList {},
14368 &mut None,
14369 )
14370 .await
14371 .unwrap();
14372 assert!(
14373 replies.is_empty(),
14374 "list reply must be detached from the connection reader"
14375 );
14376 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14377 let replies = handler
14378 .handle_client_control_request(
14379 &ctx,
14380 frame,
14381 ClientControlRequest::SupervisorSpawnSnapshot {},
14382 &mut None,
14383 )
14384 .await
14385 .unwrap();
14386 assert_eq!(
14387 replies.len(),
14388 1,
14389 "another request must complete while the subtree read stalls"
14390 );
14391 let reply = tokio::time::timeout(Duration::from_secs(5), rx.recv())
14392 .await
14393 .unwrap()
14394 .unwrap();
14395 release.send(()).unwrap();
14396 drop(release);
14397 stalled.await.unwrap();
14398 let body: Value = serde_json::from_slice(&reply.frame.body).unwrap();
14399 assert!(
14400 body["modules"][0].get("tree").is_none(),
14401 "an unknown tree reading must be omitted after the bounded read expires"
14402 );
14403 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14404 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14405 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14406 assert_eq!(
14407 body["modules"][0]["tree"]["source"], "process_tree",
14408 "the next request must read afresh"
14409 );
14410 assert_eq!(body["modules"][0]["tree"]["process_count"], 1);
14411 }
14412
14413 #[tokio::test]
14414 async fn supervisor_list_reports_loaded_config_and_omits_unconfigured() {
14415 let registry = Arc::new(Registry::default());
14416 let handle = SupervisorHandle::new();
14417 let supervisor = Supervisor::new_for_test(registry.clone(), RestartPolicy::default())
14418 .with_handle(handle.clone());
14419 supervisor
14420 .supervise_configured(
14421 crate::ModuleSpec {
14422 carry_containment: false,
14423 module_id: "drift".into(),
14424 program: PathBuf::from("disabled-module"),
14425 args: Vec::new(),
14426 env: Vec::new(),
14427 reserved: false,
14428 reserved_prefixes: Vec::new(),
14429 protocol: ModuleProtocol::Subc,
14430 overlap: Default::default(),
14431 },
14432 false,
14433 )
14434 .unwrap();
14435 let handler = ControlHandler::new(registry).with_supervisor(handle);
14436 for configured in [None, Some(false), Some(true)] {
14437 handler.capability_evaluator.configure(
14438 configured.map(|enabled| ("drift".to_string(), enabled)),
14439 BTreeMap::new(),
14440 );
14441 let frame =
14442 Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14443 let replies = handler
14444 .handle_supervisor_set_enabled(frame, "drift".into(), false, Principal::Direct)
14445 .await
14446 .unwrap();
14447 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14448 assert!(body.get("configured_enabled").is_none());
14449 assert_eq!(body["op"], "supervisor.ack");
14450 let frame =
14451 Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14452 let replies = handler.handle_supervisor_list(frame).await.unwrap();
14453 let body: Value = serde_json::from_slice(&replies[0].body).unwrap();
14454 assert_eq!(
14455 body["modules"][0].get("configured_enabled"),
14456 configured.map(Value::Bool).as_ref()
14457 );
14458 assert!(body["modules"][0].get("tree").is_none());
14459 assert!(body["modules"][0].get("subtree").is_none());
14460 }
14461 }
14462
14463 #[tokio::test(flavor = "current_thread")]
14464 async fn supervisor_set_enabled_logs_request_received_with_direct_caller() {
14465 let handler = ControlHandler::new(Arc::new(Registry::default()));
14466 let capture = EventCapture::default();
14467 let _subscriber =
14468 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14469 let (ctx, _rx) = route_ctx(ConnectionId::new(101));
14470 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14471
14472 let _ = handler
14473 .handle_client_control_request(
14474 &ctx,
14475 frame,
14476 ClientControlRequest::SupervisorSetEnabled {
14477 module_id: "broca".to_string(),
14478 enabled: false,
14479 },
14480 &mut None,
14481 )
14482 .await
14483 .unwrap();
14484
14485 let events = capture
14486 .events()
14487 .into_iter()
14488 .filter(|event| {
14489 event.target == "control"
14490 && event.fields.get("message").map(String::as_str)
14491 == Some("supervisor request received")
14492 })
14493 .collect::<Vec<_>>();
14494 assert_eq!(events.len(), 1, "one supervisor request log line");
14495 let event = &events[0];
14496 assert_eq!(event.level, tracing::Level::INFO);
14497 assert_eq!(
14498 event.fields.get("op"),
14499 Some(&format!("{:?}", ops::SUPERVISOR_SET_ENABLED))
14500 );
14501 assert_eq!(
14502 event.fields.get("module_id"),
14503 Some(&"\"broca\"".to_string())
14504 );
14505 assert_eq!(event.fields.get("enabled"), Some(&"false".to_string()));
14506 assert_eq!(event.fields.get("connection_id"), Some(&"101".to_string()));
14507 assert_eq!(event.fields.get("caller"), Some(&"direct".to_string()));
14508 }
14509
14510 #[tokio::test(flavor = "current_thread")]
14511 async fn supervisor_request_from_a_registered_module_logs_its_reserved_principal() {
14512 let registry = Arc::new(Registry::default());
14513 let module_connection = ConnectionId::new(303);
14514 registry
14515 .register_with_control_ops(
14516 manifest("aft", PROTOCOL_VERSION),
14517 PROTOCOL_VERSION,
14518 module_connection,
14519 module_baseline_control_ops(),
14520 )
14521 .unwrap();
14522 let handler = ControlHandler::new(registry);
14523 let capture = EventCapture::default();
14524 let _subscriber =
14525 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14526 let (ctx, _rx) = route_ctx(module_connection);
14527 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 1, Vec::new()).unwrap();
14528
14529 let _ = handler
14530 .handle_client_control_request(
14531 &ctx,
14532 frame,
14533 ClientControlRequest::SupervisorSetEnabled {
14534 module_id: "broca".to_string(),
14535 enabled: false,
14536 },
14537 &mut None,
14538 )
14539 .await
14540 .unwrap();
14541
14542 let callers = capture
14543 .events()
14544 .into_iter()
14545 .filter(|event| {
14546 event.fields.get("message").map(String::as_str)
14547 == Some("supervisor request received")
14548 })
14549 .map(|event| event.fields.get("caller").cloned())
14550 .collect::<Vec<_>>();
14551 assert_eq!(callers, vec![Some("reserved:aft".to_string())]);
14552 }
14553
14554 #[tokio::test(flavor = "current_thread")]
14555 async fn supervisor_list_does_not_log_request_received() {
14556 let handler = ControlHandler::new(Arc::new(Registry::default()));
14557 let capture = EventCapture::default();
14558 let _subscriber =
14559 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14560 let (ctx, _rx) = route_ctx(ConnectionId::new(102));
14561 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 2, Vec::new()).unwrap();
14562
14563 handler
14564 .handle_client_control_request(
14565 &ctx,
14566 frame,
14567 ClientControlRequest::SupervisorList {},
14568 &mut None,
14569 )
14570 .await
14571 .unwrap();
14572
14573 assert!(capture.events().into_iter().all(|event| {
14574 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14575 }));
14576 }
14577
14578 #[tokio::test(flavor = "current_thread")]
14579 async fn supervisor_rescan_preview_does_not_log_request_received() {
14580 let handler = ControlHandler::new(Arc::new(Registry::default()));
14581 let capture = EventCapture::default();
14582 let _subscriber =
14583 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14584 let (ctx, _rx) = route_ctx(ConnectionId::new(103));
14585 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 3, Vec::new()).unwrap();
14586
14587 handler
14588 .handle_client_control_request(
14589 &ctx,
14590 frame,
14591 ClientControlRequest::SupervisorRescan { preview: true },
14592 &mut None,
14593 )
14594 .await
14595 .unwrap();
14596
14597 assert!(capture.events().into_iter().all(|event| {
14598 event.fields.get("message").map(String::as_str) != Some("supervisor request received")
14599 }));
14600 }
14601
14602 struct ApproveOperator;
14603 impl crate::operator_confirm::OperatorProvider for ApproveOperator {
14604 fn prompt(
14605 &self,
14606 _: &str,
14607 _: Box<dyn FnOnce(Arc<dyn crate::operator_confirm::OperatorWithdraw>) + Send>,
14608 ) -> crate::operator_confirm::ProviderResult {
14609 crate::operator_confirm::ProviderResult::Approved
14610 }
14611 }
14612
14613 #[tokio::test]
14614 async fn privileged_operator_prompt_rescan_reports_changes_without_changing_running_mode() {
14615 assert_eq!(
14616 ControlHandler::default().privileged_operator_prompt,
14617 PrivilegedOperatorPrompt::Log
14618 );
14619 for (initial, changed, running) in [
14620 ("log", "enforce", PrivilegedOperatorPrompt::Log),
14621 ("enforce", "log", PrivilegedOperatorPrompt::Enforce),
14622 ] {
14623 let dir = ScratchDir::new("rescan-privileged-operator-prompt");
14624 let path = dir.join("subc.jsonc");
14625 std::fs::write(
14626 &path,
14627 json!({"version":1,"privileged_operator_prompt":initial}).to_string(),
14628 )
14629 .unwrap();
14630 let config = crate::daemon_config::load(&path).unwrap().unwrap();
14631 let registry = Arc::new(Registry::default());
14632 let handle = SupervisorHandle::new();
14633 let supervisor = Supervisor::new(registry.clone(), RestartPolicy::default())
14634 .with_handle(handle.clone());
14635 let handler = ControlHandler::new(registry)
14636 .with_supervisor(handle)
14637 .with_privileged_operator_prompt(config.privileged_operator_prompt)
14638 .with_supervisor_rescan(supervisor, &path, config.port);
14639 handler.forwarding.operator_confirms().configure(
14640 Arc::new(ApproveOperator),
14641 crate::operator_confirm::OperatorLimits::default(),
14642 );
14643 let (ctx, _rx) = route_ctx(ConnectionId::new(104));
14644
14645 for (disk_mode, needs_restart) in [
14646 (Some(initial), false),
14647 (Some(changed), true),
14648 (Some(changed), true),
14649 (None, running == PrivilegedOperatorPrompt::Enforce),
14650 (Some(initial), false),
14651 ] {
14652 let mut doc = json!({"version":1});
14653 if let Some(mode) = disk_mode {
14654 doc["privileged_operator_prompt"] = json!(mode);
14655 }
14656 std::fs::write(&path, doc.to_string()).unwrap();
14657 for preview in [true, false] {
14658 let request = ClientControlRequest::SupervisorRescan { preview };
14659 let frame = Frame::build(
14660 FrameType::Request,
14661 control_flags(),
14662 0,
14663 0,
14664 1,
14665 serde_json::to_vec(&request).unwrap(),
14666 )
14667 .unwrap();
14668 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14669 assert_eq!(frames.len(), 1);
14670 let ClientControlResponse::SupervisorRescan { result } =
14671 serde_json::from_slice(&frames[0].body).unwrap()
14672 else {
14673 panic!("expected a rescan response");
14674 };
14675 let expected = if needs_restart {
14676 vec!["privileged_operator_prompt"]
14677 } else {
14678 Vec::new()
14679 };
14680 assert_eq!(result.restart_required, expected);
14681 assert_eq!(handler.privileged_operator_prompt, running);
14682 assert_eq!(
14683 handler.rescan.as_ref().unwrap().privileged_operator_prompt,
14684 running
14685 );
14686 }
14687 }
14688 }
14689 }
14690
14691 #[tokio::test]
14692 async fn privileged_operator_prompt_rescan_refuses_invalid_values() {
14693 let dir = ScratchDir::new("rescan-invalid-privileged-operator-prompt");
14694 let path = dir.join("subc.jsonc");
14695 let registry = Arc::new(Registry::default());
14696 let handle = SupervisorHandle::new();
14697 let supervisor =
14698 Supervisor::new(registry.clone(), RestartPolicy::default()).with_handle(handle.clone());
14699 let handler = ControlHandler::new(registry)
14700 .with_supervisor(handle)
14701 .with_privileged_operator_prompt(PrivilegedOperatorPrompt::Enforce)
14702 .with_supervisor_rescan(supervisor, &path, None);
14703 handler.forwarding.operator_confirms().configure(
14704 Arc::new(ApproveOperator),
14705 crate::operator_confirm::OperatorLimits::default(),
14706 );
14707 let (ctx, _rx) = route_ctx(ConnectionId::new(105));
14708 for preview in [true, false] {
14709 std::fs::write(
14710 &path,
14711 r#"{"version":1,"privileged_operator_prompt":"unknown"}"#,
14712 )
14713 .unwrap();
14714 let request = ClientControlRequest::SupervisorRescan { preview };
14715 let frame = Frame::build(
14716 FrameType::Request,
14717 control_flags(),
14718 0,
14719 0,
14720 1,
14721 serde_json::to_vec(&request).unwrap(),
14722 )
14723 .unwrap();
14724 let frames = handler.handle_control_frame(&ctx, frame).await.unwrap();
14725 assert_eq!(frames.len(), 1);
14726 assert_eq!(parse_error(&frames[0])["code"], "invalid_daemon_config");
14727 assert_eq!(
14728 handler.privileged_operator_prompt,
14729 PrivilegedOperatorPrompt::Enforce
14730 );
14731 }
14732 }
14733
14734 #[tokio::test]
14739 async fn route_open_refusal_names_the_check_that_refused() {
14740 let handler = ControlHandler::new(Arc::new(Registry::default()));
14741 let capture = EventCapture::default();
14742 let _subscriber =
14743 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14744 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
14745 let response = handler
14746 .handle_control_frame(
14747 &ctx,
14748 route_open_frame(395, "nobody", unique_project_root("refusal-reason")),
14749 )
14750 .await
14751 .unwrap();
14752
14753 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14754 let event = capture
14755 .events()
14756 .into_iter()
14757 .find(|event| {
14758 event.target == "control"
14759 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
14760 })
14761 .expect("route.open refusal event");
14762 assert_eq!(
14763 event.fields.get("reason"),
14764 Some(&"\"not_registered\"".to_string())
14765 );
14766 }
14767
14768 #[tokio::test]
14769 async fn route_open_supervised_absence_emits_refusal_fields_and_counts_code() {
14770 let registry = Arc::new(Registry::default());
14771 let supervisor_handle = SupervisorHandle::new();
14772 let supervisor =
14773 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
14774 .with_handle(supervisor_handle.clone())
14775 .with_connection_file_path(std::env::temp_dir().join(format!(
14776 "subc-route-open-refusal-info-{}",
14777 std::process::id()
14778 )));
14779 let module = supervisor
14780 .supervise_configured(
14781 ModuleSpec {
14782 carry_containment: false,
14783 module_id: "warming".to_string(),
14784 program: fake_aft_stub_path(),
14785 args: Vec::new(),
14786 env: Vec::new(),
14787 reserved: false,
14788 reserved_prefixes: Vec::new(),
14789 protocol: ModuleProtocol::Subc,
14790 overlap: Default::default(),
14791 },
14792 true,
14793 )
14794 .unwrap();
14795 assert_eq!(module.state().unwrap(), ModuleState::Running);
14796
14797 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14798 assert!(handler
14799 .counters()
14800 .snapshot()
14801 .get("route_open_refused_by_code")
14802 .is_none());
14803 let capture = EventCapture::default();
14804 let _subscriber =
14805 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14806 let (ctx, _rx) = route_ctx(ConnectionId::new(94));
14807 let response = handler
14808 .handle_control_frame(
14809 &ctx,
14810 route_open_frame(394, "warming", unique_project_root("refusal-info")),
14811 )
14812 .await
14813 .unwrap();
14814 module.stop().await.unwrap();
14815
14816 assert_eq!(parse_error(&response[0])["code"], "module_warming");
14817 let event = capture
14818 .events()
14819 .into_iter()
14820 .find(|event| {
14821 event.target == "control"
14822 && event.fields.get("code") == Some(&"\"module_warming\"".to_string())
14823 })
14824 .expect("route.open refusal event");
14825 assert_eq!(
14826 event.fields.get("module_id"),
14827 Some(&"\"warming\"".to_string())
14828 );
14829 assert_eq!(event.fields.get("connection_id"), Some(&"94".to_string()));
14830 assert_eq!(
14831 event.fields.get("reason"),
14832 Some(&"\"supervised_not_registered\"".to_string())
14833 );
14834 assert_eq!(event.fields.get("state"), Some(&"running".to_string()));
14835 assert_eq!(event.fields.get("enabled"), Some(&"true".to_string()));
14836 assert_eq!(event.fields.get("live"), Some(&"false".to_string()));
14837 assert_eq!(event.fields.get("principal"), Some(&"direct".to_string()));
14838 assert_eq!(event.fields.get("scope"), Some(&"none".to_string()));
14839 assert_eq!(event.fields.get("harness"), Some(&"\"unit\"".to_string()));
14840 assert_eq!(
14841 handler.counters().snapshot()["route_open_refused_by_code"],
14842 json!({ "module_warming": 1 })
14843 );
14844 }
14845
14846 const OUTAGE_START: &str = "route.open refusing module: not serving";
14847 const OUTAGE_RECOVERED: &str = "route.open accepted again after module outage";
14848
14849 fn outage_lines(capture: &EventCapture, message: &str) -> Vec<CapturedEvent> {
14850 capture
14851 .events()
14852 .into_iter()
14853 .filter(|event| event.fields.get("message").map(String::as_str) == Some(message))
14854 .collect()
14855 }
14856
14857 fn supervise_stub(
14858 registry: &Arc<Registry>,
14859 module_id: &str,
14860 enabled: bool,
14861 ) -> (SupervisorHandle, crate::supervise::SupervisedModule) {
14862 let supervisor_handle = SupervisorHandle::new();
14863 let supervisor =
14864 Supervisor::new_for_test(Arc::clone(registry), RestartPolicy::new(0, Duration::ZERO))
14865 .with_handle(supervisor_handle.clone())
14866 .with_connection_file_path(std::env::temp_dir().join(format!(
14867 "subc-route-outage-{module_id}-{}",
14868 std::process::id()
14869 )));
14870 let module = supervisor
14871 .supervise_configured(
14872 ModuleSpec {
14873 carry_containment: false,
14874 module_id: module_id.to_string(),
14875 program: fake_aft_stub_path(),
14876 args: Vec::new(),
14877 env: Vec::new(),
14878 reserved: false,
14879 reserved_prefixes: Vec::new(),
14880 protocol: ModuleProtocol::Subc,
14881 overlap: Default::default(),
14882 },
14883 enabled,
14884 )
14885 .unwrap();
14886 (supervisor_handle, module)
14887 }
14888
14889 fn supervisor_restart_frame(corr: u64, module_id: &str) -> Frame {
14890 let body = serde_json::to_vec(&ClientControlRequest::SupervisorRestart {
14891 module_id: module_id.to_string(),
14892 drain_timeout_ms: Some(50),
14893 })
14894 .unwrap();
14895 Frame::build(FrameType::Request, control_flags(), 0, 0, corr, body).unwrap()
14896 }
14897
14898 #[test]
14902 fn handlers_over_one_forwarding_table_share_the_outage_tracker() {
14903 let registry = Arc::new(Registry::default());
14904 let forwarding = Arc::new(ForwardingTable::default());
14905 let first = ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
14906 let second = ControlHandler::with_forwarding(registry, forwarding);
14907 assert!(Arc::ptr_eq(&first.route_outages, &second.route_outages));
14908 }
14909
14910 #[tokio::test(flavor = "current_thread")]
14914 async fn route_open_unknown_module_refusals_add_no_outage_state() {
14915 let handler = ControlHandler::new(Arc::new(Registry::default()));
14916 let capture = EventCapture::default();
14917 let _subscriber =
14918 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14919 let (ctx, _rx) = route_ctx(ConnectionId::new(90));
14920 for corr in 0..8 {
14921 let response = handler
14922 .handle_control_frame(
14923 &ctx,
14924 route_open_frame(
14925 380 + corr,
14926 &format!("nobody-{corr}"),
14927 unique_project_root("outage-unknown"),
14928 ),
14929 )
14930 .await
14931 .unwrap();
14932 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
14933 }
14934
14935 assert_eq!(handler.route_outages.tracked_module_count(), 0);
14936 assert!(outage_lines(&capture, OUTAGE_START).is_empty());
14937 assert!(outage_lines(&capture, OUTAGE_RECOVERED).is_empty());
14938 }
14939
14940 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
14945 async fn route_open_outage_level_separates_operator_restart_from_unexplained() {
14946 let registry = Arc::new(Registry::default());
14947 let (supervisor_handle, module) = supervise_stub(®istry, "outage-restart", true);
14948 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
14949 let capture = EventCapture::default();
14950 let _subscriber =
14951 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
14952 let (ctx, _rx) = route_ctx(ConnectionId::new(91));
14953 handler.route_outages.record_accepted("outage-restart");
14956
14957 let response = handler
14958 .handle_control_frame(
14959 &ctx,
14960 route_open_frame(391, "outage-restart", unique_project_root("outage-a")),
14961 )
14962 .await
14963 .unwrap();
14964 assert_eq!(response[0].header.ty, FrameType::Error);
14965 let starts = outage_lines(&capture, OUTAGE_START);
14966 assert_eq!(starts.len(), 1, "{starts:?}");
14967 assert_eq!(starts[0].level, tracing::Level::WARN);
14968 assert_eq!(starts[0].fields["initiated_by"], "\"unexplained\"");
14969 assert_eq!(starts[0].fields["reason"], "\"supervised_not_registered\"");
14970 assert_eq!(starts[0].fields["module_id"], "\"outage-restart\"");
14971 handler.route_outages.record_accepted("outage-restart");
14972 assert_eq!(outage_lines(&capture, OUTAGE_RECOVERED).len(), 1);
14973
14974 let restart = handler
14975 .handle_control_frame(&ctx, supervisor_restart_frame(392, "outage-restart"))
14976 .await
14977 .unwrap();
14978 assert_eq!(
14979 restart[0].header.ty,
14980 FrameType::Response,
14981 "{:?}",
14982 parse_error(&restart[0])
14983 );
14984 handler
14985 .handle_control_frame(
14986 &ctx,
14987 route_open_frame(393, "outage-restart", unique_project_root("outage-b")),
14988 )
14989 .await
14990 .unwrap();
14991 module.stop().await.unwrap();
14992
14993 let starts = outage_lines(&capture, OUTAGE_START);
14994 assert_eq!(starts.len(), 2, "{starts:?}");
14995 assert_eq!(starts[1].level, tracing::Level::INFO);
14996 assert_eq!(starts[1].fields["initiated_by"], "\"operator\"");
14997 }
14998
14999 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
15003 async fn failed_operator_restart_leaves_no_operator_mark() {
15004 let registry = Arc::new(Registry::default());
15005 let (supervisor_handle, _module) = supervise_stub(®istry, "outage-disabled", false);
15006 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15007 let capture = EventCapture::default();
15008 let _subscriber =
15009 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15010 let (ctx, _rx) = route_ctx(ConnectionId::new(92));
15011 handler.route_outages.record_accepted("outage-disabled");
15012
15013 let restart = handler
15014 .handle_control_frame(&ctx, supervisor_restart_frame(394, "outage-disabled"))
15015 .await
15016 .unwrap();
15017 assert_eq!(parse_error(&restart[0])["code"], "module_disabled");
15018 assert!(!handler.route_outages.has_operator_mark("outage-disabled"));
15019
15020 handler
15021 .handle_control_frame(
15022 &ctx,
15023 route_open_frame(395, "outage-disabled", unique_project_root("outage-c")),
15024 )
15025 .await
15026 .unwrap();
15027 let starts = outage_lines(&capture, OUTAGE_START);
15028 assert_eq!(starts.len(), 1, "{starts:?}");
15029 assert_eq!(starts[0].level, tracing::Level::WARN);
15030 }
15031
15032 #[tokio::test(flavor = "current_thread")]
15033 async fn route_open_unknown_module_escapes_target_module_id() {
15034 let handler = ControlHandler::new(Arc::new(Registry::default()));
15035 let capture = EventCapture::default();
15036 let _subscriber =
15037 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15038 let hostile_module_id = "\u{1b}]52;c;AAAA\u{07}";
15039 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15040 let response = handler
15041 .handle_control_frame(
15042 &ctx,
15043 route_open_frame(
15044 395,
15045 hostile_module_id,
15046 unique_project_root("hostile-target-module-id"),
15047 ),
15048 )
15049 .await
15050 .unwrap();
15051
15052 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15053 let event = capture
15054 .events()
15055 .into_iter()
15056 .find(|event| {
15057 event.target == "control"
15058 && event.fields.get("code") == Some(&"\"unknown_module\"".to_string())
15059 })
15060 .expect("route.open unknown-module refusal event");
15061 let logged = event.fields.get("module_id").expect("module_id field");
15062 assert!(!logged.bytes().any(|byte| byte < 0x20));
15063 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
15064 }
15065
15066 #[tokio::test(flavor = "current_thread")]
15069 async fn route_open_refusal_carries_daemon_origin() {
15070 let handler = ControlHandler::new(Arc::new(Registry::default()));
15071 let (ctx, _rx) = route_ctx(ConnectionId::new(95));
15072 let response = handler
15073 .handle_control_frame(
15074 &ctx,
15075 route_open_frame(395, "nobody", unique_project_root("refusal-origin")),
15076 )
15077 .await
15078 .unwrap();
15079
15080 assert_eq!(response.len(), 1);
15081 assert_eq!(response[0].header.ty, FrameType::Error);
15082 assert_eq!(parse_error(&response[0])["code"], "unknown_module");
15083 assert!(
15084 response[0].header.flags.is_daemon_origin(),
15085 "route.open refusal must carry daemon origin: {:?}",
15086 response[0].header.flags
15087 );
15088 }
15089
15090 #[tokio::test(flavor = "current_thread")]
15094 async fn route_open_to_draining_module_refuses_module_reloading_with_daemon_origin() {
15095 let registry = Arc::new(Registry::default());
15096 let forwarding = Arc::new(ForwardingTable::default());
15097 let handler =
15098 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15099 let module_connection = ConnectionId::new(95);
15100 let (module_ctx, mut module_rx) = route_ctx(module_connection);
15101 hello_via_sink(
15102 &handler,
15103 &module_ctx,
15104 &mut module_rx,
15105 hello_frame("aft", PROTOCOL_VERSION, 395),
15106 )
15107 .await;
15108 forwarding
15109 .begin_module_drain("aft", RouteCloseReason::Restart)
15110 .unwrap()
15111 .expect("the module endpoint drains");
15112
15113 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(96));
15114 let response = handler
15115 .handle_control_frame(
15116 &client_ctx,
15117 route_open_frame(396, "aft", unique_project_root("drain-origin")),
15118 )
15119 .await
15120 .unwrap();
15121
15122 assert_eq!(response.len(), 1);
15123 assert_eq!(response[0].header.ty, FrameType::Error);
15124 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15125 assert!(
15126 response[0].header.flags.is_daemon_origin(),
15127 "drain module_reloading must carry daemon origin: {:?}",
15128 response[0].header.flags
15129 );
15130 }
15131
15132 #[tokio::test(flavor = "current_thread")]
15136 async fn route_open_module_rejection_with_daemon_code_carries_no_daemon_origin() {
15137 let registry = Arc::new(Registry::default());
15138 let forwarding = Arc::new(ForwardingTable::default());
15139 let handler =
15140 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15141 let module_connection = ConnectionId::new(95);
15142 let (module_ctx, mut module_rx) = route_ctx(module_connection);
15143 hello_via_sink(
15144 &handler,
15145 &module_ctx,
15146 &mut module_rx,
15147 hello_frame("aft", PROTOCOL_VERSION, 395),
15148 )
15149 .await;
15150
15151 let client_connection = ConnectionId::new(96);
15152 let (client_ctx, _client_rx) = route_ctx(client_connection);
15153 let (route_task, bind) = relay_route_open(
15154 &handler,
15155 client_connection,
15156 &client_ctx.egress,
15157 &mut module_rx,
15158 396,
15159 "aft",
15160 "spoofed-reloading",
15161 )
15162 .await;
15163 let rejection = Frame::build(
15166 FrameType::Error,
15167 crate::router::daemon_error_flags(),
15168 0,
15169 0,
15170 bind.header.corr,
15171 serde_json::to_vec(&ErrorBody::new(
15172 "module_reloading",
15173 "module refused route.bind",
15174 ))
15175 .unwrap(),
15176 )
15177 .unwrap();
15178 handler
15179 .handle_control_frame(&module_ctx, rejection)
15180 .await
15181 .unwrap();
15182
15183 let response = route_task.await.unwrap();
15184 assert_eq!(response.len(), 1);
15185 assert_eq!(response[0].header.ty, FrameType::Error);
15186 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
15187 assert!(
15188 !response[0].header.flags.is_daemon_origin(),
15189 "a module's own rejection must not carry daemon origin: {:?}",
15190 response[0].header.flags
15191 );
15192 }
15193
15194 #[tokio::test(flavor = "current_thread")]
15195 async fn route_open_module_rejection_uses_daemon_counter_key() {
15196 let registry = Arc::new(Registry::default());
15197 let forwarding = Arc::new(ForwardingTable::default());
15198 let handler =
15199 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15200 let module_connection = ConnectionId::new(95);
15201 let (module_ctx, mut module_rx) = route_ctx(module_connection);
15202 hello_via_sink(
15203 &handler,
15204 &module_ctx,
15205 &mut module_rx,
15206 hello_frame("aft", PROTOCOL_VERSION, 395),
15207 )
15208 .await;
15209
15210 let client_connection = ConnectionId::new(96);
15211 let (client_ctx, _client_rx) = route_ctx(client_connection);
15212 let capture = EventCapture::default();
15213 let _subscriber =
15214 tracing::subscriber::set_default(tracing_subscriber::registry().with(capture.clone()));
15215 let (route_task, bind) = relay_route_open(
15216 &handler,
15217 client_connection,
15218 &client_ctx.egress,
15219 &mut module_rx,
15220 396,
15221 "aft",
15222 "hostile-module-code",
15223 )
15224 .await;
15225 let hostile_code = "\u{1b}]52;c;AAAA\u{07}";
15226 let rejection = Frame::build(
15227 FrameType::Error,
15228 control_flags(),
15229 0,
15230 0,
15231 bind.header.corr,
15232 serde_json::to_vec(&ErrorBody::new(hostile_code, "module refused route.bind")).unwrap(),
15233 )
15234 .unwrap();
15235 handler
15236 .handle_control_frame(&module_ctx, rejection)
15237 .await
15238 .unwrap();
15239
15240 let response = route_task.await.unwrap();
15241 assert_eq!(parse_error(&response[0])["code"], hostile_code);
15242 let counters = handler.counters().snapshot();
15243 assert_eq!(
15244 counters["route_open_refused_by_code"],
15245 json!({ "module_rejected": 1 })
15246 );
15247 assert!(counters["route_open_refused_by_code"]
15248 .get(hostile_code)
15249 .is_none());
15250
15251 let event = capture
15252 .events()
15253 .into_iter()
15254 .find(|event| {
15255 event.target == "control"
15256 && event.fields.get("code") == Some(&"\"module_rejected\"".to_string())
15257 })
15258 .expect("route.open module-rejection refusal event");
15259 let logged = event.fields.get("module_code").expect("module_code field");
15260 assert!(!logged.bytes().any(|byte| byte < 0x20));
15261 assert_eq!(logged, r#""\u{1b}]52;c;AAAA\u{7}""#);
15262 }
15263
15264 #[tokio::test]
15265 async fn route_open_keeps_failed_unregistered_supervised_module_unavailable() {
15266 let registry = Arc::new(Registry::default());
15267 let supervisor_handle = SupervisorHandle::new();
15268 let missing_program = std::env::temp_dir().join(format!(
15269 "subc-route-open-missing-program-{}",
15270 std::process::id()
15271 ));
15272 let supervisor =
15273 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::new(0, Duration::ZERO))
15274 .with_handle(supervisor_handle.clone());
15275 let module = supervisor
15276 .supervise_configured(
15277 ModuleSpec {
15278 carry_containment: false,
15279 module_id: "failed".to_string(),
15280 program: missing_program,
15281 args: Vec::new(),
15282 env: Vec::new(),
15283 reserved: false,
15284 reserved_prefixes: Vec::new(),
15285 protocol: ModuleProtocol::Subc,
15286 overlap: Default::default(),
15287 },
15288 true,
15289 )
15290 .unwrap();
15291 assert_eq!(module.state().unwrap(), ModuleState::Failed);
15292
15293 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
15294 let (ctx, _rx) = route_ctx(ConnectionId::new(40));
15295 let response = handler
15296 .handle_control_frame(
15297 &ctx,
15298 route_open_frame(305, "failed", unique_project_root("failed")),
15299 )
15300 .await
15301 .unwrap();
15302
15303 assert_eq!(response[0].header.ty, FrameType::Error);
15304 let error = parse_error(&response[0]);
15305 assert_eq!(error["code"], "target_unavailable");
15306 assert!(error["message"]
15307 .as_str()
15308 .unwrap()
15309 .contains("state=failed, enabled=true, live=false"));
15310 }
15311
15312 #[tokio::test]
15313 async fn route_open_role_mismatch_remains_target_unavailable() {
15314 let registry = Arc::new(Registry::default());
15315 let handler = ControlHandler::new(Arc::clone(®istry));
15316 handler
15317 .handle_control(
15318 ConnectionId::new(41),
15319 non_routable_hello_frame_with_control_ops("health-only", 306, None),
15320 )
15321 .unwrap();
15322
15323 let (ctx, _rx) = route_ctx(ConnectionId::new(42));
15324 let response = handler
15325 .handle_control_frame(
15326 &ctx,
15327 route_open_frame(307, "health-only", unique_project_root("role-mismatch")),
15328 )
15329 .await
15330 .unwrap();
15331
15332 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
15333 assert!(parse_error(&response[0])["message"]
15334 .as_str()
15335 .unwrap()
15336 .contains("does not provide the requested target"));
15337 }
15338
15339 #[tokio::test]
15340 async fn route_open_inactive_registration_remains_target_unavailable() {
15341 let registry = Arc::new(Registry::default());
15342 let handler = ControlHandler::new(Arc::clone(®istry));
15343 handler
15344 .handle_control(
15345 ConnectionId::new(43),
15346 hello_frame("inactive", PROTOCOL_VERSION, 308),
15347 )
15348 .unwrap();
15349 assert!(registry
15350 .set_module_state_for_test("inactive", ChannelState::Closed)
15351 .unwrap());
15352
15353 let (ctx, _rx) = route_ctx(ConnectionId::new(44));
15354 let response = handler
15355 .handle_control_frame(
15356 &ctx,
15357 route_open_frame(309, "inactive", unique_project_root("inactive")),
15358 )
15359 .await
15360 .unwrap();
15361
15362 assert_eq!(parse_error(&response[0])["code"], "target_unavailable");
15363 assert!(parse_error(&response[0])["message"]
15364 .as_str()
15365 .unwrap()
15366 .contains("is not active"));
15367 }
15368
15369 #[tokio::test]
15370 async fn late_health_reply_is_recorded_through_the_module_response_path() {
15371 let registry = Arc::new(Registry::default());
15372 let forwarding = Arc::new(ForwardingTable::default());
15373 let supervisor_handle = SupervisorHandle::new();
15374 let supervisor =
15375 Supervisor::new_for_test(Arc::clone(®istry), crate::RestartPolicy::default())
15376 .with_forwarding(Arc::clone(&forwarding))
15377 .with_handle(supervisor_handle.clone());
15378 let module = supervisor
15379 .supervise_configured(
15380 crate::ModuleSpec {
15381 carry_containment: false,
15382 module_id: "late-health-response".to_string(),
15383 program: PathBuf::from("disabled-module"),
15384 args: Vec::new(),
15385 env: Vec::new(),
15386 reserved: false,
15387 reserved_prefixes: Vec::new(),
15388 protocol: ModuleProtocol::Subc,
15389 overlap: Default::default(),
15390 },
15391 false,
15392 )
15393 .unwrap();
15394 let handler =
15395 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15396 .with_supervisor(supervisor_handle);
15397 let (module_ctx, _module_rx) = route_ctx(ConnectionId::new(39));
15398 handler
15399 .handle_control_frame(
15400 &module_ctx,
15401 hello_frame_with_control_ops(
15402 "late-health-response",
15403 PROTOCOL_VERSION,
15404 7,
15405 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15406 ),
15407 )
15408 .await
15409 .unwrap();
15410 let probe_started_at = Instant::now() - Duration::from_millis(80);
15411 let pending = forwarding
15412 .begin_health_probe_rpc_for(
15413 "late-health-response",
15414 MODULE_CONTROL_OP_HEALTH_CHECK,
15415 probe_started_at,
15416 Instant::now() - Duration::from_millis(1),
15417 )
15418 .unwrap();
15419 assert!(forwarding
15420 .tombstone_health_probe_rpc(pending.endpoint, pending.corr)
15421 .unwrap());
15422
15423 let responses = handler
15424 .handle_control_frame(&module_ctx, health_response(pending.corr, HealthStatus::Ok))
15425 .await
15426 .unwrap();
15427
15428 assert!(responses.is_empty());
15429 let health = module.status().unwrap().health;
15430 assert_eq!(health.late_answer_count, 1);
15431 assert!(health.last_late_answer_latency_ms.unwrap() >= 80);
15432 }
15433
15434 #[tokio::test]
15435 async fn health_probe_timeout_and_module_death_are_typed() {
15436 let registry = Arc::new(Registry::default());
15437 let forwarding = Arc::new(ForwardingTable::default());
15438 let handler =
15439 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15440 .with_health_probe_timeout(Duration::from_millis(50));
15441 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(40));
15442 hello_via_sink(
15443 &handler,
15444 &module_ctx,
15445 &mut module_rx,
15446 hello_frame_with_control_ops(
15447 "aft",
15448 PROTOCOL_VERSION,
15449 7,
15450 Some(vec![MODULE_CONTROL_OP_HEALTH_CHECK.to_string()]),
15451 ),
15452 )
15453 .await;
15454
15455 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(41));
15456 let responses = handler
15457 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(201, "aft"))
15458 .await
15459 .unwrap();
15460 assert_eq!(responses[0].header.ty, FrameType::Error);
15461 assert_eq!(parse_error(&responses[0])["code"], "module_timeout");
15462 let _ = module_rx.try_recv();
15463
15464 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(42));
15465 let health_handler = handler.clone();
15466 let death_task = tokio::spawn(async move {
15467 health_handler
15468 .handle_control_frame(&client_ctx, supervisor_health_probe_frame(202, "aft"))
15469 .await
15470 .unwrap()
15471 });
15472 tokio::time::timeout(Duration::from_secs(1), module_rx.recv())
15473 .await
15474 .unwrap()
15475 .unwrap();
15476 handler
15477 .cleanup_connection(module_ctx.connection_id)
15478 .unwrap();
15479 let responses = death_task.await.unwrap();
15480 assert_eq!(responses[0].header.ty, FrameType::Error);
15481 assert_eq!(parse_error(&responses[0])["code"], "target_unavailable");
15482 }
15483
15484 #[test]
15485 fn hello_requires_exact_protocol_version() {
15486 for (connection, offered) in [(1, PROTOCOL_VERSION - 1), (2, PROTOCOL_VERSION + 1)] {
15487 let registry = Arc::new(Registry::default());
15488 let handler = ControlHandler::new(Arc::clone(®istry));
15489 let responses = handler
15490 .handle_control(
15491 ConnectionId::new(connection),
15492 hello_frame("aft", offered, 9),
15493 )
15494 .unwrap();
15495
15496 assert_eq!(responses.len(), 1);
15497 assert_eq!(responses[0].header.ty, FrameType::Error);
15498 let error = parse_error(&responses[0]);
15499 assert_eq!(error["code"], "version_unsupported");
15500 assert!(registry.get_module("aft").unwrap().is_none());
15501 assert_eq!(registry.active_registration_count().unwrap(), 0);
15502 }
15503 }
15504
15505 #[test]
15506 fn unknown_module_push_op_is_ignored_but_malformed_known_op_errors() {
15507 let registry = Arc::new(Registry::default());
15508 let forwarding = Arc::new(ForwardingTable::default());
15509 let handler =
15510 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15511 let module_connection = ConnectionId::new(301);
15512 let registration = registry
15513 .register_with_control_ops(
15514 manifest("aft-push", PROTOCOL_VERSION),
15515 PROTOCOL_VERSION,
15516 module_connection,
15517 module_baseline_control_ops(),
15518 )
15519 .unwrap();
15520 let (module_tx, _module_rx) = mpsc::channel(8);
15521 let endpoint = forwarding
15522 .register_module_connection(
15523 module_connection,
15524 "aft-push".to_string(),
15525 PROTOCOL_VERSION,
15526 manifest_concurrency(®istration.manifest),
15527 FrameSink::new(module_tx),
15528 )
15529 .unwrap();
15530
15531 let unknown = Frame::build(
15533 FrameType::Push,
15534 control_flags(),
15535 0,
15536 0,
15537 5,
15538 serde_json::to_vec(&json!({"op": "route.future.v2", "extra": 1})).unwrap(),
15539 )
15540 .unwrap();
15541 let out = handler.handle_status_update(endpoint, unknown).unwrap();
15542 assert!(
15543 out.is_empty(),
15544 "unknown push op must be ignored, got {out:?}"
15545 );
15546
15547 let malformed = Frame::build(
15549 FrameType::Push,
15550 control_flags(),
15551 0,
15552 0,
15553 6,
15554 serde_json::to_vec(&json!({"op": "route.status"})).unwrap(),
15555 )
15556 .unwrap();
15557 let out = handler.handle_status_update(endpoint, malformed).unwrap();
15558 assert_eq!(out.len(), 1);
15559 assert_eq!(out[0].header.ty, FrameType::Error);
15560 assert_eq!(parse_error(&out[0])["code"], "invalid_control_body");
15561 }
15562
15563 #[test]
15564 fn hello_rejected_when_connection_already_owns_client_routes() {
15565 let registry = Arc::new(Registry::default());
15566 let forwarding = Arc::new(ForwardingTable::default());
15567 let handler =
15568 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15569 let _ = bind_liveness_route(®istry, &forwarding, "aft-module");
15571 let client_connection = ConnectionId::new(202);
15572
15573 let responses = handler
15576 .handle_control(
15577 client_connection,
15578 hello_frame("aft-second", PROTOCOL_VERSION, 9),
15579 )
15580 .unwrap();
15581 assert_eq!(responses[0].header.ty, FrameType::Error);
15582 assert_eq!(parse_error(&responses[0])["code"], "invalid_hello");
15583 assert!(registry.get_module("aft-second").unwrap().is_none());
15584 }
15585
15586 #[tokio::test]
15587 async fn second_hello_preserves_registration_routes_and_launch_nonce() {
15588 let registry = Arc::new(Registry::default());
15589 let forwarding = Arc::new(ForwardingTable::default());
15590 let handler = ControlHandler::with_forwarding(registry.clone(), forwarding.clone());
15591 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(101));
15592 hello_via_sink(
15593 &handler,
15594 &module_ctx,
15595 &mut module_rx,
15596 hello_frame_with_nonce("alpha", PROTOCOL_VERSION, 1, Some("alpha-nonce")),
15597 )
15598 .await;
15599 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(202));
15600 let pending = forwarding
15601 .begin_route_bind_relay_for_test(
15602 client_ctx.connection_id,
15603 client_ctx.egress.clone(),
15604 2,
15605 "alpha",
15606 )
15607 .unwrap();
15608 forwarding
15609 .complete_pending_relay(
15610 module_ctx.connection_id,
15611 pending.corr,
15612 RouteBindRelayOutcome::Accepted,
15613 )
15614 .unwrap();
15615 client_rx.try_recv().unwrap();
15616 for module_id in ["beta", "alpha"] {
15617 let replies = handler
15618 .handle_control_frame(
15619 &module_ctx,
15620 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 3, Some("replacement")),
15621 )
15622 .await
15623 .unwrap();
15624 assert_eq!(replies.len(), 1, "second HELLO must be refused");
15625 assert_eq!(parse_error(&replies[0])["code"], "invalid_hello");
15626 }
15627 assert_eq!(registry.list_modules().unwrap().1.len(), 1);
15628 assert!(registry.get_module("beta").unwrap().is_none());
15629 assert!(matches!(
15630 forwarding
15631 .lookup_data_route(
15632 client_ctx.connection_id,
15633 pending.client_channel,
15634 pending.client_epoch,
15635 )
15636 .unwrap(),
15637 DataRoute::Client(DataRouteState::Bound(_))
15638 ));
15639 assert!(handler
15640 .hello_launch_nonces
15641 .lock()
15642 .unwrap()
15643 .presented(module_ctx.connection_id, Some("alpha-nonce")));
15644 assert!(module_rx.try_recv().is_err());
15645 }
15646
15647 #[test]
15648 fn reserved_module_hello_requires_matching_launch_nonce() {
15649 let registry = Arc::new(Registry::default());
15650 let supervisor = SupervisorHandle::new();
15651 supervisor.set_reserved_nonce("vault", "the-real-nonce".to_string());
15654 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15655
15656 let no_nonce = handler
15658 .handle_control(
15659 ConnectionId::new(1),
15660 hello_frame("vault", PROTOCOL_VERSION, 1),
15661 )
15662 .unwrap();
15663 assert_eq!(no_nonce[0].header.ty, FrameType::Error);
15664 assert_eq!(parse_error(&no_nonce[0])["code"], "reserved_module");
15665 assert!(registry.get_module("vault").unwrap().is_none());
15666
15667 let wrong = handler
15669 .handle_control(
15670 ConnectionId::new(2),
15671 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some("forged")),
15672 )
15673 .unwrap();
15674 assert_eq!(wrong[0].header.ty, FrameType::Error);
15675 assert_eq!(parse_error(&wrong[0])["code"], "reserved_module");
15676 assert!(registry.get_module("vault").unwrap().is_none());
15677
15678 let ok = handler
15680 .handle_control(
15681 ConnectionId::new(3),
15682 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some("the-real-nonce")),
15683 )
15684 .unwrap();
15685 assert_eq!(ok[0].header.ty, FrameType::HelloAck);
15686 assert!(registry.get_module("vault").unwrap().is_some());
15687 }
15688
15689 #[test]
15690 fn reserved_prefix_hello_uses_delimiter_sensitive_owner_nonce() {
15691 let registry = Arc::new(Registry::default());
15692 let supervisor = SupervisorHandle::new();
15693 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15694 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15695 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15696
15697 let squat = handler
15698 .handle_control(
15699 ConnectionId::new(1),
15700 hello_frame("fed:peerA:tool", PROTOCOL_VERSION, 1),
15701 )
15702 .unwrap();
15703 assert_eq!(squat[0].header.ty, FrameType::Error);
15704 assert_eq!(parse_error(&squat[0])["code"], "reserved_module");
15705 assert!(parse_error(&squat[0])["message"]
15706 .as_str()
15707 .unwrap()
15708 .contains("fed:"));
15709
15710 let accepted_peer = handler
15711 .handle_control(
15712 ConnectionId::new(2),
15713 hello_frame_with_nonce("fed:peerA:tool", PROTOCOL_VERSION, 2, Some("owner-nonce")),
15714 )
15715 .unwrap();
15716 assert_eq!(accepted_peer[0].header.ty, FrameType::HelloAck);
15717
15718 let accepted_short = handler
15719 .handle_control(
15720 ConnectionId::new(3),
15721 hello_frame_with_nonce("fed:x", PROTOCOL_VERSION, 3, Some("owner-nonce")),
15722 )
15723 .unwrap();
15724 assert_eq!(accepted_short[0].header.ty, FrameType::HelloAck);
15725
15726 for (conn, module_id) in [(4, "fedx:tool"), (5, "fed"), (6, "FED:x")] {
15727 let response = handler
15728 .handle_control(
15729 ConnectionId::new(conn),
15730 hello_frame(module_id, PROTOCOL_VERSION, conn),
15731 )
15732 .unwrap();
15733 assert_eq!(response[0].header.ty, FrameType::HelloAck, "{module_id}");
15734 }
15735 }
15736
15737 #[test]
15738 fn exact_reserved_module_takes_precedence_over_reserved_prefix() {
15739 let registry = Arc::new(Registry::default());
15740 let supervisor = SupervisorHandle::new();
15741 supervisor.set_spawn_nonce("federation", "owner-nonce".to_string());
15742 supervisor.set_reserved_prefixes("federation", &["fed:".to_string()]);
15743 supervisor.set_reserved_nonce("fed:special", "exact-nonce".to_string());
15744 let handler = ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor);
15745
15746 let owner_nonce = handler
15747 .handle_control(
15748 ConnectionId::new(1),
15749 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 1, Some("owner-nonce")),
15750 )
15751 .unwrap();
15752 assert_eq!(owner_nonce[0].header.ty, FrameType::Error);
15753 assert_eq!(parse_error(&owner_nonce[0])["code"], "reserved_module");
15754 assert!(registry.get_module("fed:special").unwrap().is_none());
15755
15756 let exact_nonce = handler
15757 .handle_control(
15758 ConnectionId::new(2),
15759 hello_frame_with_nonce("fed:special", PROTOCOL_VERSION, 2, Some("exact-nonce")),
15760 )
15761 .unwrap();
15762 assert_eq!(exact_nonce[0].header.ty, FrameType::HelloAck);
15763 assert!(registry.get_module("fed:special").unwrap().is_some());
15764 }
15765
15766 #[test]
15767 fn non_reserved_module_ignores_launch_nonce() {
15768 let registry = Arc::new(Registry::default());
15769 let handler = ControlHandler::new(Arc::clone(®istry));
15772 let no_nonce = handler
15773 .handle_control(
15774 ConnectionId::new(1),
15775 hello_frame("aft-no-nonce", PROTOCOL_VERSION, 1),
15776 )
15777 .unwrap();
15778 assert_eq!(no_nonce[0].header.ty, FrameType::HelloAck);
15779 assert!(registry.get_module("aft-no-nonce").unwrap().is_some());
15780
15781 let echoed_nonce = handler
15782 .handle_control(
15783 ConnectionId::new(2),
15784 hello_frame_with_nonce("aft-with-nonce", PROTOCOL_VERSION, 2, Some("spawn-nonce")),
15785 )
15786 .unwrap();
15787 assert_eq!(echoed_nonce[0].header.ty, FrameType::HelloAck);
15788 assert!(registry.get_module("aft-with-nonce").unwrap().is_some());
15789 }
15790
15791 #[test]
15792 fn malformed_hello_returns_error_and_handler_still_answers_ping() {
15793 let handler = ControlHandler::default();
15794 let conn = ConnectionId::new(1);
15795 let malformed = Frame::build(
15796 FrameType::Hello,
15797 control_flags(),
15798 0,
15799 0,
15800 3,
15801 b"{not json".to_vec(),
15802 )
15803 .unwrap();
15804
15805 let error = handler.handle_control(conn, malformed).unwrap();
15806 assert_eq!(error[0].header.ty, FrameType::Error);
15807 assert_eq!(parse_error(&error[0])["code"], "invalid_hello");
15808
15809 let ping = Frame::build(FrameType::Ping, control_flags(), 0, 0, 4, Vec::new()).unwrap();
15810 let pong = handler.handle_control(conn, ping).unwrap();
15811 assert_eq!(pong[0].header.ty, FrameType::Pong);
15812 assert_eq!(pong[0].header.corr, 4);
15813 }
15814
15815 #[test]
15816 fn duplicate_module_id_is_rejected_without_replacing_active_registration() {
15817 let registry = Arc::new(Registry::default());
15818 let handler = ControlHandler::new(Arc::clone(®istry));
15819
15820 handler
15821 .handle_control(
15822 ConnectionId::new(1),
15823 hello_frame("aft", PROTOCOL_VERSION, 1),
15824 )
15825 .unwrap();
15826 let duplicate = handler
15827 .handle_control(
15828 ConnectionId::new(2),
15829 hello_frame("aft", PROTOCOL_VERSION, 2),
15830 )
15831 .unwrap();
15832
15833 assert_eq!(duplicate[0].header.ty, FrameType::Error);
15834 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
15835 let registration = registry.get_module("aft").unwrap().unwrap();
15836 assert_eq!(registration.connection_id, ConnectionId::new(1));
15837 }
15838
15839 #[test]
15840 fn liveness_poll_reports_false_when_process_liveness_reports_dead() {
15841 let registry = Arc::new(Registry::default());
15842 let forwarding = Arc::new(ForwardingTable::default());
15843 let process_liveness = Arc::new(FakeProcessLiveness { live: Some(false) });
15844 let handler =
15845 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15846 .with_process_liveness(process_liveness);
15847 let (ctx, route_channel, route_epoch) =
15848 bind_liveness_route(®istry, &forwarding, "aft-dead");
15849 let responses = handler
15850 .handle_route_poll(
15851 &ctx,
15852 route_poll_frame(41, PollKind::Liveness, route_channel),
15853 route_channel,
15854 route_epoch,
15855 PollKind::Liveness,
15856 )
15857 .unwrap();
15858
15859 assert_eq!(responses.len(), 1);
15860 assert_eq!(responses[0].header.ty, FrameType::Response);
15861 assert_route_poll_liveness(&responses[0], false);
15862 }
15863
15864 #[test]
15865 fn liveness_poll_without_process_source_uses_bound_route() {
15866 let registry = Arc::new(Registry::default());
15867 let forwarding = Arc::new(ForwardingTable::default());
15868 let handler =
15869 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
15870 let (ctx, route_channel, route_epoch) =
15871 bind_liveness_route(®istry, &forwarding, "aft-bound-only");
15872 let responses = handler
15873 .handle_route_poll(
15874 &ctx,
15875 route_poll_frame(42, PollKind::Liveness, route_channel),
15876 route_channel,
15877 route_epoch,
15878 PollKind::Liveness,
15879 )
15880 .unwrap();
15881
15882 assert_route_poll_liveness(&responses[0], true);
15883 }
15884
15885 #[test]
15886 fn liveness_poll_untracked_process_source_uses_bound_route() {
15887 let registry = Arc::new(Registry::default());
15888 let forwarding = Arc::new(ForwardingTable::default());
15889 let process_liveness = Arc::new(FakeProcessLiveness { live: None });
15890 let handler =
15891 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
15892 .with_process_liveness(process_liveness);
15893 let (ctx, route_channel, route_epoch) =
15894 bind_liveness_route(®istry, &forwarding, "aft-untracked");
15895 let responses = handler
15896 .handle_route_poll(
15897 &ctx,
15898 route_poll_frame(43, PollKind::Liveness, route_channel),
15899 route_channel,
15900 route_epoch,
15901 PollKind::Liveness,
15902 )
15903 .unwrap();
15904
15905 assert_route_poll_liveness(&responses[0], true);
15906 }
15907
15908 #[tokio::test]
15909 async fn unknown_op_returns_unknown_control_op() {
15910 let handler = ControlHandler::default();
15911 let (ctx, _rx) = route_ctx(ConnectionId::new(77));
15912 let request = Frame::build(
15913 FrameType::Request,
15914 control_flags(),
15915 0,
15916 0,
15917 55,
15918 br#"{"op":"route.nope","route_channel":1}"#.to_vec(),
15919 )
15920 .unwrap();
15921
15922 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15923
15924 assert_eq!(response.len(), 1);
15925 assert_eq!(response[0].header.ty, FrameType::Error);
15926 assert_eq!(response[0].header.corr, 55);
15927 assert_eq!(parse_error(&response[0])["code"], "unknown_control_op");
15928 }
15929
15930 #[tokio::test]
15931 async fn supervisor_provenance_rejects_unknown_exact_module() {
15932 let handler = ControlHandler::default();
15933 let (ctx, _rx) = route_ctx(ConnectionId::new(79));
15934 let request = Frame::build(
15935 FrameType::Request,
15936 control_flags(),
15937 0,
15938 0,
15939 57,
15940 br#"{"op":"supervisor.provenance","module_id":"missing"}"#.to_vec(),
15941 )
15942 .unwrap();
15943
15944 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
15945
15946 assert_eq!(response.len(), 1);
15947 assert_eq!(response[0].header.ty, FrameType::Error);
15948 assert_eq!(response[0].header.corr, 57);
15949 let error = parse_error(&response[0]);
15950 assert_eq!(error["code"], "unknown_module");
15951 assert_eq!(error["message"], "module_id 'missing' is not supervised");
15952 }
15953
15954 #[test]
15955 fn provenance_probe_override_keeps_handler_tests_deterministic() {
15956 let expected = subc_control::RunningImageAgreement::Unavailable {
15957 reason: subc_control::RunningImageUnavailableReason::HashFailed,
15958 };
15959 let handler = ControlHandler::default().with_provenance_probe_result(expected.clone());
15960 assert_eq!(handler.provenance_probe_override, Some(expected));
15961 }
15962
15963 #[test]
15964 fn reload_verdict_detects_configured_program_different_from_spawned_path() {
15965 let verdict = reload_verdict(
15966 std::path::Path::new("/bin/new"),
15967 Some(std::path::Path::new("/bin/old")),
15968 subc_control::RunningImageAgreement::Unavailable {
15969 reason: subc_control::RunningImageUnavailableReason::HashFailed,
15970 },
15971 );
15972 assert!(matches!(
15973 verdict.path,
15974 subc_control::ReloadPathAgreement::Mismatch { configured, spawned_from }
15975 if configured == std::path::Path::new("/bin/new")
15976 && spawned_from == std::path::Path::new("/bin/old")
15977 ));
15978 }
15979
15980 #[test]
15981 fn launch_spec_without_running_digest_is_unavailable_not_match() {
15982 use subc_control::{
15983 LaunchSpecAgreement as Agreement, LaunchSpecUnavailableReason as Reason,
15984 };
15985 let configured = crate::supervise::ModuleSpec {
15986 carry_containment: false,
15987 module_id: "legacy".into(),
15988 program: "/module".into(),
15989 args: Vec::new(),
15990 env: Vec::new(),
15991 reserved: false,
15992 reserved_prefixes: Vec::new(),
15993 protocol: subc_control::ModuleProtocol::Subc,
15994 overlap: Default::default(),
15995 };
15996 let digest = crate::supervise::LaunchSpecDigest::from_spec(&configured);
15997 assert_eq!(
15998 launch_spec_agreement(&configured, false, Some(&digest)),
15999 Agreement::Unavailable {
16000 reason: Reason::NotRunning
16001 }
16002 );
16003 assert_eq!(
16004 launch_spec_agreement(&configured, true, None),
16005 Agreement::Unavailable {
16006 reason: Reason::SpawnedSpecUnavailable
16007 }
16008 );
16009 assert_eq!(
16010 launch_spec_agreement(&configured, true, Some(&digest)),
16011 Agreement::Match
16012 );
16013 }
16014
16015 #[cfg(windows)]
16016 #[test]
16017 fn windows_reload_path_matches_configured_and_verbatim_spawn_spellings() {
16018 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-spelling");
16019 let configured = dir.join("module.exe");
16020 std::fs::write(&configured, b"executable path fixture").unwrap();
16021 let spawned = std::fs::canonicalize(&configured).unwrap();
16022 assert_ne!(
16023 configured, spawned,
16024 "fixture must exercise different path spellings"
16025 );
16026 let forward_slashes = configured.to_string_lossy().replace('\\', "/");
16027 let lower_case = configured.to_string_lossy().to_lowercase();
16028 for spelling in [
16029 &configured,
16030 Path::new(&forward_slashes),
16031 Path::new(&lower_case),
16032 ] {
16033 let verdict = reload_verdict(
16034 spelling,
16035 Some(&spawned),
16036 subc_control::RunningImageAgreement::Unavailable {
16037 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16038 },
16039 );
16040 assert_eq!(
16041 verdict.path,
16042 ReloadPathAgreement::Match,
16043 "configured {spelling:?}, spawned {spawned:?}"
16044 );
16045 }
16046 }
16047
16048 #[cfg(windows)]
16049 #[test]
16050 fn windows_reload_path_keeps_different_existing_program_pending() {
16051 let dir = cortexkit_test_support::ScratchDir::new("windows-reload-path-change");
16052 let old = dir.join("old.exe");
16053 let new = dir.join("new.exe");
16054 std::fs::write(&old, b"same executable content").unwrap();
16055 std::fs::copy(&old, &new).unwrap();
16056 let spawned = std::fs::canonicalize(&old).unwrap();
16057 let image = subc_control::RunningImageAgreement::Unavailable {
16058 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16059 };
16060 let verdict = reload_verdict(&new, Some(&spawned), image.clone());
16061 assert!(
16062 matches!(verdict.path, ReloadPathAgreement::Mismatch { configured, spawned_from }
16063 if configured == new && spawned_from == spawned)
16064 );
16065 let missing = dir.join("missing.exe");
16066 assert!(matches!(
16067 reload_verdict(&missing, Some(&spawned), image).path,
16068 ReloadPathAgreement::Mismatch { .. }
16069 ));
16070 }
16071
16072 #[test]
16073 fn reload_verdict_detects_replaced_image_at_same_path() {
16074 let image = subc_control::RunningImageAgreement::Mismatch {
16075 running: subc_control::RunningImageEvidence::LinuxProcSha256 {
16076 digest: "old".into(),
16077 },
16078 disk: subc_control::RunningImageEvidence::LinuxProcSha256 {
16079 digest: "new".into(),
16080 },
16081 };
16082 let verdict = reload_verdict(
16083 std::path::Path::new("/bin/same"),
16084 Some(std::path::Path::new("/bin/same")),
16085 image.clone(),
16086 );
16087 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16088 assert_eq!(verdict.image, image);
16089 }
16090
16091 #[test]
16092 fn reload_verdict_preserves_stopped_and_unavailable_reasons() {
16093 let image = subc_control::RunningImageAgreement::Unavailable {
16094 reason: subc_control::RunningImageUnavailableReason::NotRunning,
16095 };
16096 let verdict = reload_verdict(std::path::Path::new("/bin/same"), None, image.clone());
16097 assert_eq!(
16098 verdict.path,
16099 subc_control::ReloadPathAgreement::Unavailable {
16100 reason: subc_control::ReloadPathUnavailableReason::NotRunning,
16101 }
16102 );
16103 assert_eq!(verdict.image, image);
16104
16105 let unconfirmed = subc_control::RunningImageAgreement::Unavailable {
16106 reason: subc_control::RunningImageUnavailableReason::ProcessIdentityUnconfirmed,
16107 };
16108 let verdict = reload_verdict(
16109 std::path::Path::new("/bin/same"),
16110 Some(std::path::Path::new("/bin/same")),
16111 unconfirmed.clone(),
16112 );
16113 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16114 assert_eq!(verdict.image, unconfirmed);
16115 }
16116
16117 #[test]
16118 fn reload_verdict_preserves_each_image_unavailability_reason() {
16119 use subc_control::RunningImageUnavailableReason as Reason;
16120
16121 for reason in [
16122 Reason::NotRunning,
16123 Reason::UnsupportedPlatform,
16124 Reason::RunningExecutableUnreadable,
16125 Reason::SpawnedPathUnreadable,
16126 Reason::HashFailed,
16127 Reason::ProcessIdentityUnconfirmed,
16128 Reason::Unknown("future_probe_reason".to_string()),
16129 ] {
16130 let image = subc_control::RunningImageAgreement::Unavailable {
16131 reason: reason.clone(),
16132 };
16133 let verdict = reload_verdict(
16134 std::path::Path::new("/bin/same"),
16135 Some(std::path::Path::new("/bin/same")),
16136 image.clone(),
16137 );
16138 assert_eq!(verdict.path, subc_control::ReloadPathAgreement::Match);
16139 assert_eq!(verdict.image, image, "{reason:?}");
16140 }
16141 }
16142
16143 #[tokio::test]
16144 async fn malformed_control_bodies_return_invalid_control_body() {
16145 let handler = ControlHandler::default();
16146 let (ctx, _rx) = route_ctx(ConnectionId::new(78));
16147
16148 for (corr, body) in [
16149 (56, br#"{"route_channel":1}"#.as_slice()),
16150 (57, br#"{"op":17,"route_channel":1}"#.as_slice()),
16151 (
16152 58,
16153 br#"{"op":"route.poll","route_channel":"bad","kind":"status"}"#.as_slice(),
16154 ),
16155 ] {
16156 let request = Frame::build(
16157 FrameType::Request,
16158 control_flags(),
16159 0,
16160 0,
16161 corr,
16162 body.to_vec(),
16163 )
16164 .unwrap();
16165 let response = handler.handle_control_frame(&ctx, request).await.unwrap();
16166
16167 assert_eq!(response.len(), 1);
16168 assert_eq!(response[0].header.ty, FrameType::Error);
16169 assert_eq!(response[0].header.corr, corr);
16170 assert_eq!(parse_error(&response[0])["code"], "invalid_control_body");
16171 }
16172 }
16173
16174 #[tokio::test]
16175 async fn goodbye_tears_down_registration_and_later_channel_is_unknown() {
16176 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16177 let registry = Arc::new(Registry::default());
16178 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
16179 let router = Router::with_control_handler(Arc::clone(&control));
16180 let connection = router.begin_connection();
16181 let (ctx, mut rx) = route_ctx(connection.id());
16182
16183 router
16184 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 11))
16185 .await
16186 .unwrap();
16187 let response = rx.recv().await.unwrap();
16188 let ack = parse_ack(&response);
16189 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16190 let channel = 1;
16191
16192 let goodbye =
16193 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 12, Vec::new()).unwrap();
16194 router.route_for_connection(&ctx, goodbye).await.unwrap();
16195 assert!(rx.try_recv().is_err());
16196 assert!(registry.get_module("aft").unwrap().is_none());
16197
16198 router
16199 .route_for_connection(&ctx, channel_request(channel, 13))
16200 .await
16201 .unwrap();
16202 let error_frame = rx.recv().await.unwrap();
16203 assert_eq!(error_frame.header.ty, FrameType::Error);
16204 assert_eq!(error_frame.header.channel, channel);
16205 let captured = crate::router::test_log::captured_logs(&logs);
16206 assert_eq!(
16207 captured
16208 .lines()
16209 .filter(|line| {
16210 line.contains("module_id=aft")
16211 && line.contains("reason=explicit_goodbye")
16212 && line.contains("module registration ended")
16213 })
16214 .count(),
16215 1,
16216 "unexpected GOODBYE registry log: {captured}"
16217 );
16218 }
16219
16220 #[tokio::test]
16221 async fn module_goodbye_refreshes_requirements_and_pushes_route_closed() {
16222 let registry = Arc::new(Registry::default());
16223 let handler = ControlHandler::new(registry).with_capability_config(
16224 [("prov".to_string(), true), ("cons".to_string(), true)],
16225 BTreeMap::new(),
16226 );
16227 let (provider_ctx, mut provider_rx) = route_ctx(ConnectionId::new(701));
16228 register_capability_manifest(
16229 &handler,
16230 &provider_ctx,
16231 &mut provider_rx,
16232 capability_manifest("prov", &["thing/v1"], &[]),
16233 1,
16234 )
16235 .await;
16236 let mut consumer = capability_manifest("cons", &[], &[]);
16237 consumer.capabilities.as_mut().unwrap().requires.push(
16238 subc_protocol::manifest::CapabilityRequirement {
16239 capability: "thing/v1".to_string(),
16240 need: subc_protocol::manifest::CapabilityNeed::Required,
16241 },
16242 );
16243 let (consumer_ctx, mut consumer_rx) = route_ctx(ConnectionId::new(702));
16244 register_capability_manifest(&handler, &consumer_ctx, &mut consumer_rx, consumer, 2).await;
16245 assert_eq!(
16246 handler.capability_evaluator.verdict("cons", "thing/v1"),
16247 Some(CapabilityVerdict::Provided)
16248 );
16249 let (mut client_rx, _) = open_route_for_capability_test(
16250 &handler,
16251 &provider_ctx,
16252 &mut provider_rx,
16253 703,
16254 3,
16255 "prov",
16256 None,
16257 )
16258 .await;
16259 let goodbye =
16260 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap();
16261 handler
16262 .handle_control_frame(&provider_ctx, goodbye)
16263 .await
16264 .unwrap();
16265 assert_eq!(
16266 handler.capability_evaluator.verdict("cons", "thing/v1"),
16267 Some(CapabilityVerdict::NeverProvided)
16268 );
16269 let closed = client_rx
16270 .try_recv()
16271 .expect("GOODBYE pushes route.closed before route GOODBYE");
16272 assert!(
16273 matches!(serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap(),
16274 ClientControlPush::RouteClosed { module_id, channels, .. } if module_id == "prov" && channels.len() == 1)
16275 );
16276 assert_eq!(client_rx.try_recv().unwrap().header.ty, FrameType::Goodbye);
16277 assert_eq!(handler.forwarding.active_binding_count().unwrap(), 0);
16278 }
16279
16280 struct RouteClosedFixture {
16281 handler: ControlHandler,
16282 #[cfg(unix)]
16283 supervisor: Supervisor,
16284 module_ctx: RouteCtx,
16285 #[cfg(unix)]
16286 module_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16287 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
16288 route: Arc<crate::forwarding::RouteBinding>,
16289 }
16290
16291 async fn route_closed_fixture() -> RouteClosedFixture {
16292 let registry = Arc::new(Registry::default());
16293 let handler = ControlHandler::new(Arc::clone(®istry));
16294 #[cfg(unix)]
16295 let supervisor = Supervisor::new(registry, RestartPolicy::default())
16296 .with_forwarding(Arc::clone(&handler.forwarding));
16297 let (module_ctx, mut module_rx) = route_ctx(ConnectionId::new(701));
16298 register_capability_manifest(
16299 &handler,
16300 &module_ctx,
16301 &mut module_rx,
16302 capability_manifest("prov", &[], &[]),
16303 1,
16304 )
16305 .await;
16306 let (client_rx, bind) = open_route_for_capability_test(
16307 &handler,
16308 &module_ctx,
16309 &mut module_rx,
16310 702,
16311 2,
16312 "prov",
16313 None,
16314 )
16315 .await;
16316 let ModuleControlRequest::RouteBind {
16317 route_channel: channel,
16318 epoch: route_epoch,
16319 ..
16320 } = bind
16321 else {
16322 panic!("expected route.bind");
16323 };
16324 let crate::forwarding::DataRoute::Module(crate::forwarding::DataRouteState::Bound(route)) =
16325 handler
16326 .forwarding
16327 .lookup_data_route(module_ctx.connection_id, channel, route_epoch)
16328 .unwrap()
16329 else {
16330 panic!("the acknowledged route must be bound");
16331 };
16332 RouteClosedFixture {
16333 handler,
16334 #[cfg(unix)]
16335 supervisor,
16336 module_ctx,
16337 #[cfg(unix)]
16338 module_rx,
16339 client_rx,
16340 route,
16341 }
16342 }
16343
16344 impl RouteClosedFixture {
16345 #[cfg(unix)]
16346 async fn cut(&mut self) {
16347 let notices = async {
16348 let mut draining = self.module_rx.recv().await.unwrap();
16349 assert!(matches!(
16350 serde_json::from_slice::<subc_protocol::session::ModuleControlCommand>(
16351 &draining.body
16352 )
16353 .unwrap(),
16354 subc_protocol::session::ModuleControlCommand::Draining {
16355 reason: RouteCloseReason::Disable,
16356 ..
16357 }
16358 ));
16359 draining.flushed.take().unwrap().send(()).unwrap();
16360 let mut closing = self.client_rx.recv().await.unwrap();
16361 assert!(matches!(
16362 serde_json::from_slice::<ClientControlPush>(&closing.body).unwrap(),
16363 ClientControlPush::RouteClosing {
16364 reason: RouteCloseReason::Restart,
16365 ..
16366 }
16367 ));
16368 closing.flushed.take().unwrap().send(()).unwrap();
16369 };
16370 let (result, ()) = tokio::join!(self.supervisor.drain_for_daemon_shutdown(), notices);
16371 result.unwrap();
16372 }
16373
16374 fn disconnect(&mut self) -> ClientControlPush {
16375 self.handler
16376 .cleanup_connection(self.module_ctx.connection_id)
16377 .unwrap();
16378 let closed = self.client_rx.try_recv().unwrap();
16379 assert_eq!(closed.header.ty, FrameType::Push);
16380 let push = serde_json::from_slice::<ClientControlPush>(&closed.body).unwrap();
16381 assert!(matches!(
16382 &push,
16383 ClientControlPush::RouteClosed { channels, abandoned: 0, .. }
16384 if channels == &[self.route.client_channel]
16385 ));
16386 assert_eq!(
16387 self.client_rx.try_recv().unwrap().header.ty,
16388 FrameType::Goodbye
16389 );
16390 assert_eq!(self.handler.forwarding.active_binding_count().unwrap(), 0);
16391 push
16392 }
16393 }
16394
16395 #[cfg(unix)]
16396 #[tokio::test(start_paused = true)]
16397 async fn daemon_cut_quiet_routes_report_restart_and_drained() {
16398 let mut fixture = route_closed_fixture().await;
16399 fixture.cut().await;
16400 assert!(matches!(
16401 fixture.disconnect(),
16402 ClientControlPush::RouteClosed {
16403 reason: RouteCloseReason::Restart,
16404 drained: true,
16405 ..
16406 }
16407 ));
16408 }
16409
16410 #[cfg(unix)]
16411 #[tokio::test(start_paused = true)]
16412 async fn daemon_cut_in_flight_past_budget_reports_restart_and_not_drained() {
16413 let mut fixture = route_closed_fixture().await;
16414 fixture.route.flow.acquire_tagged(99, false).await.unwrap();
16415 let started = tokio::time::Instant::now();
16416 fixture.cut().await;
16417 assert!(started.elapsed() >= Duration::from_secs(2));
16418 assert_eq!(fixture.route.flow.in_flight(), 1);
16419 assert!(matches!(
16420 fixture.disconnect(),
16421 ClientControlPush::RouteClosed {
16422 reason: RouteCloseReason::Restart,
16423 drained: false,
16424 ..
16425 }
16426 ));
16427 }
16428
16429 #[tokio::test]
16430 async fn module_connection_crash_reports_crash_and_not_drained() {
16431 let mut fixture = route_closed_fixture().await;
16432 assert!(matches!(
16436 fixture.disconnect(),
16437 ClientControlPush::RouteClosed {
16438 reason: RouteCloseReason::Crash,
16439 drained: false,
16440 ..
16441 }
16442 ));
16443 }
16444
16445 #[tokio::test]
16446 async fn dropping_router_connection_releases_registration() {
16447 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16448 let registry = Arc::new(Registry::default());
16449 let control = Arc::new(ControlHandler::new(Arc::clone(®istry)));
16450 let router = Router::with_control_handler(Arc::clone(&control));
16451 let connection = router.begin_connection();
16452 let connection_id = connection.id();
16453 let (ctx, mut rx) = route_ctx(connection_id);
16454
16455 router
16456 .route_for_connection(&ctx, hello_frame("aft", PROTOCOL_VERSION, 31))
16457 .await
16458 .unwrap();
16459 let response = rx.recv().await.unwrap();
16460 let ack = parse_ack(&response);
16461 assert_eq!(ack.negotiated_ver, PROTOCOL_VERSION);
16462 assert!(registry.get_module("aft").unwrap().is_some());
16463
16464 drop(connection);
16465
16466 assert!(registry.get_module("aft").unwrap().is_none());
16467 assert_eq!(registry.active_registration_count().unwrap(), 0);
16468
16469 control
16470 .cleanup_connection(ConnectionId::new(u64::MAX))
16471 .unwrap();
16472 let captured = crate::router::test_log::captured_logs(&logs);
16473 println!("captured registry lifecycle logs:\n{captured}");
16474 let events: Vec<_> = captured
16475 .lines()
16476 .filter(|line| {
16477 line.contains("module registered module_id=aft ")
16478 || line.contains("module registration ended")
16479 })
16480 .collect();
16481 assert_eq!(
16482 events.len(),
16483 2,
16484 "unexpected registry lifecycle logs: {captured}"
16485 );
16486 assert!(events[0].contains("module registered module_id=aft "));
16487 assert!(events[0].contains(&format!("connection_id={}", connection_id.get())));
16488 assert!(events[1].contains(&format!(
16489 "module_id=aft connection_id={}",
16490 connection_id.get()
16491 )));
16492 assert!(events[1].contains("reason=connection_closed"));
16493 assert!(events[1].contains("module registration ended"));
16494 assert_eq!(
16495 captured
16496 .lines()
16497 .filter(|line| line.contains("module registered module_id=aft "))
16498 .count(),
16499 1,
16500 "legacy registration admission line must appear once: {captured}"
16501 );
16502 }
16503
16504 #[tokio::test]
16505 async fn hello_registration_keeps_legacy_module_registered_line_once() {
16506 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
16507 let registry = Arc::new(Registry::default());
16508 let control = ControlHandler::new(Arc::clone(®istry));
16509 let (ctx, mut rx) = route_ctx(ConnectionId::new(777));
16510 hello_via_sink(
16511 &control,
16512 &ctx,
16513 &mut rx,
16514 hello_frame("prefrontal-host:test", PROTOCOL_VERSION, 1),
16515 )
16516 .await;
16517
16518 let captured = crate::router::test_log::captured_logs(&logs);
16519 let admissions: Vec<_> = captured
16520 .lines()
16521 .filter(|line| line.contains("module registered module_id=prefrontal-host:test "))
16522 .collect();
16523 assert_eq!(
16524 admissions.len(),
16525 1,
16526 "legacy admission line must remain exactly once: {captured}"
16527 );
16528 assert!(admissions[0].contains("routable_provider=true"));
16529 assert!(admissions[0].contains("connection_id=777"));
16530 }
16531
16532 fn capability_manifest(
16533 module_id: &str,
16534 provides: &[&str],
16535 must_never_reach: &[&str],
16536 ) -> ModuleManifest {
16537 let mut manifest = manifest(module_id, PROTOCOL_VERSION);
16538 manifest.capabilities = Some(CapabilityDeclarations {
16539 provides: provides
16540 .iter()
16541 .map(|capability| (*capability).to_string())
16542 .collect(),
16543 requires: Vec::new(),
16544 must_never_reach: must_never_reach
16545 .iter()
16546 .map(|capability| (*capability).to_string())
16547 .collect(),
16548 });
16549 manifest
16550 }
16551
16552 fn hello_frame_with_manifest(manifest: ModuleManifest, corr: u64) -> Frame {
16553 Frame::build(
16554 FrameType::Hello,
16555 control_flags(),
16556 0,
16557 0,
16558 corr,
16559 serde_json::to_vec(&ModuleHelloBody {
16560 protocol_ver: manifest.protocol_ver,
16561 manifest,
16562 control_ops: None,
16563 launch_nonce: None,
16564 })
16565 .expect("capability test HELLO serializes"),
16566 )
16567 .expect("capability test HELLO frame builds")
16568 }
16569
16570 fn catalog_update_with_capabilities_frame(
16571 corr: u64,
16572 capabilities: CapabilityDeclarations,
16573 ) -> Frame {
16574 Frame::build(
16575 FrameType::Request,
16576 control_flags(),
16577 0,
16578 0,
16579 corr,
16580 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
16581 provides: manifest("catalog-update-placeholder", PROTOCOL_VERSION).provides,
16582 capabilities: Some(capabilities),
16583 ready: None,
16584 })
16585 .expect("capability catalog.update serializes"),
16586 )
16587 .expect("capability catalog.update frame builds")
16588 }
16589
16590 async fn register_capability_manifest(
16591 handler: &ControlHandler,
16592 ctx: &RouteCtx,
16593 rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16594 manifest: ModuleManifest,
16595 corr: u64,
16596 ) {
16597 hello_via_sink(handler, ctx, rx, hello_frame_with_manifest(manifest, corr)).await;
16598 }
16599
16600 async fn open_route_for_capability_test(
16601 handler: &ControlHandler,
16602 target_ctx: &RouteCtx,
16603 target_rx: &mut mpsc::Receiver<crate::router::OutboundFrame>,
16604 client_connection_id: u64,
16605 corr: u64,
16606 target_module_id: &str,
16607 consumer_identity: Option<ConsumerIdentity>,
16608 ) -> (
16609 mpsc::Receiver<crate::router::OutboundFrame>,
16610 ModuleControlRequest,
16611 ) {
16612 let (client_ctx, mut client_rx) = route_ctx(ConnectionId::new(client_connection_id));
16613 let route_handler = handler.clone();
16614 let target_module_id = target_module_id.to_string();
16615 let route_task = tokio::spawn(async move {
16616 route_handler
16617 .handle_control_frame(
16618 &client_ctx,
16619 route_open_frame_with_admission_facts(
16620 corr,
16621 &target_module_id,
16622 unique_project_root("admission-facts"),
16623 consumer_identity,
16624 None,
16625 ),
16626 )
16627 .await
16628 .expect("capability test route.open succeeds")
16629 });
16630 let bind = tokio::time::timeout(Duration::from_secs(1), target_rx.recv())
16631 .await
16632 .expect("capability test route.open must reach route.bind")
16633 .expect("target control receiver stays open");
16634 let bind_request: ModuleControlRequest =
16635 serde_json::from_slice(&bind.body).expect("route.bind decodes");
16636 handler
16637 .handle_control_frame(target_ctx, route_bind_ack(bind.header.corr))
16638 .await
16639 .expect("capability test route.bind ACK succeeds");
16640 assert!(route_task.await.expect("route.open task joins").is_empty());
16641 let opened = client_rx
16642 .recv()
16643 .await
16644 .expect("successful route.open publishes a response");
16645 assert!(matches!(
16646 serde_json::from_slice::<ClientControlResponse>(&opened.body),
16647 Ok(ClientControlResponse::RouteOpen { .. })
16648 ));
16649 (client_rx, bind_request)
16650 }
16651
16652 fn assert_capability_denied_push(frame: Frame, target_module_id: &str) -> u16 {
16653 assert_eq!(frame.header.ty, FrameType::Push);
16654 assert_eq!(frame.header.channel, 0);
16655 let push = serde_json::from_slice::<ClientControlPush>(&frame.body)
16656 .expect("route.closed control push decodes");
16657 let ClientControlPush::RouteClosed { channels, .. } = &push else {
16658 panic!("expected route.closed");
16659 };
16660 assert_eq!(channels.len(), 1, "exactly one violating route closed");
16661 let closed_channel = channels[0];
16662 let channels = channels.clone();
16663 assert_eq!(
16664 push,
16665 ClientControlPush::RouteClosed {
16666 module_id: target_module_id.to_string(),
16667 channels,
16668 reason: RouteCloseReason::CapabilityDenied,
16669 drained: false,
16670 abandoned: 0,
16671 excluded_subscriptions: 0,
16672 terminal: Some(false),
16673 }
16674 );
16675 closed_channel
16676 }
16677
16678 #[tokio::test]
16679 async fn route_open_capability_forbidden_mutation_proof_creates_no_route() {
16680 let registry = Arc::new(Registry::default());
16681 let forwarding = Arc::new(ForwardingTable::default());
16682 let supervisor = SupervisorHandle::new();
16683 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16684 let handler =
16685 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16686 .with_supervisor(supervisor);
16687 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(700));
16688 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(701));
16689 register_capability_manifest(
16690 &handler,
16691 &target_ctx,
16692 &mut target_rx,
16693 capability_manifest("target", &["credentials-provider/v1"], &[]),
16694 1,
16695 )
16696 .await;
16697 register_capability_manifest(
16698 &handler,
16699 &opener_ctx,
16700 &mut opener_rx,
16701 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16702 2,
16703 )
16704 .await;
16705
16706 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(702));
16707 let replies = handler
16708 .handle_control_frame(
16709 &client_ctx,
16710 route_open_frame_with_admission_facts(
16711 3,
16712 "target",
16713 unique_project_root("admission-facts"),
16714 Some(ConsumerIdentity {
16715 module_id: "opener".to_string(),
16716 launch_nonce: "opener-nonce".to_string(),
16717 }),
16718 None,
16719 ),
16720 )
16721 .await
16722 .expect("denied route.open returns a typed frame");
16723 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
16724 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16725 assert!(
16726 target_rx.try_recv().is_err(),
16727 "forbidden route.open must not relay route.bind"
16728 );
16729 }
16730
16731 #[tokio::test]
16732 async fn capability_deny_edge_hello_mutation_proof_force_closes_existing_route() {
16733 let registry = Arc::new(Registry::default());
16734 let forwarding = Arc::new(ForwardingTable::default());
16735 let supervisor = SupervisorHandle::new();
16736 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16737 let handler =
16738 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16739 .with_supervisor(supervisor);
16740 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(710));
16741 let (old_opener_ctx, mut old_opener_rx) = route_ctx(ConnectionId::new(711));
16742 register_capability_manifest(
16743 &handler,
16744 &target_ctx,
16745 &mut target_rx,
16746 capability_manifest("target", &["credentials-provider/v1"], &[]),
16747 1,
16748 )
16749 .await;
16750 register_capability_manifest(
16751 &handler,
16752 &old_opener_ctx,
16753 &mut old_opener_rx,
16754 capability_manifest("opener", &[], &[]),
16755 2,
16756 )
16757 .await;
16758 let (mut client_rx, _) = open_route_for_capability_test(
16759 &handler,
16760 &target_ctx,
16761 &mut target_rx,
16762 712,
16763 3,
16764 "target",
16765 Some(ConsumerIdentity {
16766 module_id: "opener".to_string(),
16767 launch_nonce: "opener-nonce".to_string(),
16768 }),
16769 )
16770 .await;
16771 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16772
16773 handler
16774 .cleanup_connection(old_opener_ctx.connection_id)
16775 .expect("old opener registration cleans up");
16776 let (new_opener_ctx, mut new_opener_rx) = route_ctx(ConnectionId::new(713));
16777 register_capability_manifest(
16778 &handler,
16779 &new_opener_ctx,
16780 &mut new_opener_rx,
16781 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16782 4,
16783 )
16784 .await;
16785
16786 let closed_channel = assert_capability_denied_push(
16787 client_rx
16788 .try_recv()
16789 .expect("HELLO deny addition must emit route.closed")
16790 .frame,
16791 "target",
16792 );
16793 let goodbye = client_rx
16794 .try_recv()
16795 .expect("HELLO deny addition must send the client a route GOODBYE")
16796 .frame;
16797 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16798 assert_eq!(goodbye.header.channel, closed_channel);
16799 assert_ne!(goodbye.header.channel, 0);
16800 assert_ne!(goodbye.header.epoch, 0);
16801 assert!(client_rx.try_recv().is_err());
16802 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16803 assert!(matches!(
16804 target_rx.try_recv(),
16805 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16806 ));
16807 }
16808
16809 #[tokio::test]
16810 async fn capability_claim_catalog_update_mutation_proof_force_closes_existing_route() {
16811 let registry = Arc::new(Registry::default());
16812 let forwarding = Arc::new(ForwardingTable::default());
16813 let supervisor = SupervisorHandle::new();
16814 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16815 let handler =
16816 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16817 .with_supervisor(supervisor);
16818 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(720));
16819 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(721));
16820 register_capability_manifest(
16821 &handler,
16822 &target_ctx,
16823 &mut target_rx,
16824 capability_manifest("target", &[], &[]),
16825 1,
16826 )
16827 .await;
16828 register_capability_manifest(
16829 &handler,
16830 &opener_ctx,
16831 &mut opener_rx,
16832 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16833 2,
16834 )
16835 .await;
16836 let (mut client_rx, _) = open_route_for_capability_test(
16837 &handler,
16838 &target_ctx,
16839 &mut target_rx,
16840 722,
16841 3,
16842 "target",
16843 Some(ConsumerIdentity {
16844 module_id: "opener".to_string(),
16845 launch_nonce: "opener-nonce".to_string(),
16846 }),
16847 )
16848 .await;
16849 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16850
16851 let replies = handler
16852 .handle_control_frame(
16853 &target_ctx,
16854 catalog_update_with_capabilities_frame(
16855 4,
16856 CapabilityDeclarations {
16857 provides: vec!["credentials-provider/v1".to_string()],
16858 requires: Vec::new(),
16859 must_never_reach: Vec::new(),
16860 },
16861 ),
16862 )
16863 .await
16864 .expect("claim catalog.update succeeds");
16865 assert!(matches!(
16866 serde_json::from_slice::<ModuleControlResponseToModule>(&replies[0].body),
16867 Ok(ModuleControlResponseToModule::CatalogUpdate {})
16868 ));
16869 let closed_channel = assert_capability_denied_push(
16870 client_rx
16871 .try_recv()
16872 .expect("claim addition must emit route.closed")
16873 .frame,
16874 "target",
16875 );
16876 let goodbye = client_rx
16877 .try_recv()
16878 .expect("claim addition must send the client a route GOODBYE")
16879 .frame;
16880 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
16881 assert_eq!(goodbye.header.channel, closed_channel);
16882 assert_ne!(goodbye.header.channel, 0);
16883 assert_ne!(goodbye.header.epoch, 0);
16884 assert!(client_rx.try_recv().is_err());
16885 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
16886 assert!(matches!(
16887 target_rx.try_recv(),
16888 Ok(outbound) if outbound.header.ty == FrameType::Goodbye
16889 ));
16890 }
16891
16892 #[tokio::test]
16893 async fn capability_claim_removal_mutation_proof_keeps_route_open_without_close_frame() {
16894 let registry = Arc::new(Registry::default());
16895 let forwarding = Arc::new(ForwardingTable::default());
16896 let supervisor = SupervisorHandle::new();
16897 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16898 let handler =
16899 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16900 .with_supervisor(supervisor);
16901 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(730));
16902 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(731));
16903 register_capability_manifest(
16904 &handler,
16905 &target_ctx,
16906 &mut target_rx,
16907 capability_manifest("target", &["credentials-provider/v1"], &[]),
16908 1,
16909 )
16910 .await;
16911 register_capability_manifest(
16912 &handler,
16913 &opener_ctx,
16914 &mut opener_rx,
16915 capability_manifest("opener", &[], &[]),
16916 2,
16917 )
16918 .await;
16919 let (mut client_rx, _) = open_route_for_capability_test(
16920 &handler,
16921 &target_ctx,
16922 &mut target_rx,
16923 732,
16924 3,
16925 "target",
16926 Some(ConsumerIdentity {
16927 module_id: "opener".to_string(),
16928 launch_nonce: "opener-nonce".to_string(),
16929 }),
16930 )
16931 .await;
16932 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
16933
16934 handler
16935 .handle_control_frame(
16936 &target_ctx,
16937 catalog_update_with_capabilities_frame(
16938 4,
16939 CapabilityDeclarations {
16940 provides: Vec::new(),
16941 requires: Vec::new(),
16942 must_never_reach: Vec::new(),
16943 },
16944 ),
16945 )
16946 .await
16947 .expect("claim removal catalog.update succeeds");
16948 assert_eq!(
16949 forwarding.active_binding_count().unwrap(),
16950 1,
16951 "removing an attested target claim must leave the route census unchanged"
16952 );
16953 assert!(
16954 client_rx.try_recv().is_err(),
16955 "claim removal must not emit route.closed or a client route GOODBYE"
16956 );
16957 assert!(
16958 target_rx.try_recv().is_err(),
16959 "claim removal must not send the target a route GOODBYE"
16960 );
16961 }
16962
16963 #[tokio::test]
16966 async fn direct_client_scope_honesty_mutation_proof_opens_denied_capability_provider() {
16967 let registry = Arc::new(Registry::default());
16968 let forwarding = Arc::new(ForwardingTable::default());
16969 let supervisor = SupervisorHandle::new();
16970 supervisor.set_spawn_nonce("opener", "opener-nonce".to_string());
16971 let handler =
16972 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
16973 .with_supervisor(supervisor);
16974 let (target_ctx, mut target_rx) = route_ctx(ConnectionId::new(740));
16975 let (opener_ctx, mut opener_rx) = route_ctx(ConnectionId::new(741));
16976 register_capability_manifest(
16977 &handler,
16978 &target_ctx,
16979 &mut target_rx,
16980 capability_manifest("target", &["credentials-provider/v1"], &[]),
16981 1,
16982 )
16983 .await;
16984 register_capability_manifest(
16985 &handler,
16986 &opener_ctx,
16987 &mut opener_rx,
16988 capability_manifest("opener", &[], &["credentials-provider/v1"]),
16989 2,
16990 )
16991 .await;
16992
16993 let (_client_rx, bind) = open_route_for_capability_test(
16994 &handler,
16995 &target_ctx,
16996 &mut target_rx,
16997 742,
16998 3,
16999 "target",
17000 None,
17001 )
17002 .await;
17003 let ModuleControlRequest::RouteBind { principal, .. } = bind else {
17004 panic!("direct scope-honesty route must bind");
17005 };
17006 assert_eq!(principal, Some(Principal::Direct));
17007 assert_eq!(forwarding.active_binding_count().unwrap(), 1);
17008 }
17009
17010 #[tokio::test]
17013 async fn must_never_reach_self_route_is_capability_forbidden() {
17014 let registry = Arc::new(Registry::default());
17015 let forwarding = Arc::new(ForwardingTable::default());
17016 let supervisor = SupervisorHandle::new();
17017 supervisor.set_spawn_nonce("self-provider", "self-nonce".to_string());
17018 let handler =
17019 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
17020 .with_supervisor(supervisor);
17021 let (self_ctx, mut self_rx) = route_ctx(ConnectionId::new(750));
17022 register_capability_manifest(
17023 &handler,
17024 &self_ctx,
17025 &mut self_rx,
17026 capability_manifest(
17027 "self-provider",
17028 &["credentials-provider/v1"],
17029 &["credentials-provider/v1"],
17030 ),
17031 1,
17032 )
17033 .await;
17034
17035 let (client_ctx, _client_rx) = route_ctx(ConnectionId::new(751));
17036 let replies = handler
17037 .handle_control_frame(
17038 &client_ctx,
17039 route_open_frame_with_admission_facts(
17040 2,
17041 "self-provider",
17042 unique_project_root("admission-facts"),
17043 Some(ConsumerIdentity {
17044 module_id: "self-provider".to_string(),
17045 launch_nonce: "self-nonce".to_string(),
17046 }),
17047 None,
17048 ),
17049 )
17050 .await
17051 .expect("self-route refusal returns a typed frame");
17052 assert_eq!(parse_error(&replies[0])["code"], "capability_forbidden");
17053 assert_eq!(forwarding.active_binding_count().unwrap(), 0);
17054 assert!(
17055 self_rx.try_recv().is_err(),
17056 "self denial must not relay route.bind"
17057 );
17058 }
17059
17060 #[test]
17061 fn unsupported_channel_zero_frame_returns_error() {
17062 let handler = ControlHandler::default();
17063 let request = Frame::build(
17064 FrameType::Request,
17065 control_flags(),
17066 0,
17067 0,
17068 21,
17069 b"opaque".to_vec(),
17070 )
17071 .unwrap();
17072
17073 let response = handler
17074 .handle_control(ConnectionId::new(1), request)
17075 .unwrap();
17076
17077 assert_eq!(response[0].header.ty, FrameType::Error);
17078 assert_eq!(
17079 parse_error(&response[0])["code"],
17080 "unsupported_control_frame"
17081 );
17082 }
17083
17084 mod swap {
17089 use super::*;
17090
17091 const INCUMBENT: ConnectionId = ConnectionId::new(30);
17092 const CANDIDATE: ConnectionId = ConnectionId::new(40);
17093
17094 struct Swap {
17095 registry: Arc<Registry>,
17096 forwarding: Arc<ForwardingTable>,
17097 handler: ControlHandler,
17098 incumbent_ctx: RouteCtx,
17099 incumbent_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17100 candidate_ctx: RouteCtx,
17101 candidate_rx: mpsc::Receiver<crate::router::OutboundFrame>,
17102 }
17103
17104 async fn swap_with_incumbent() -> Swap {
17105 let registry = Arc::new(Registry::default());
17106 let forwarding = Arc::new(ForwardingTable::default());
17107 let handler =
17108 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding));
17109 let (incumbent_ctx, mut incumbent_rx) = route_ctx(INCUMBENT);
17110 hello_via_sink(
17111 &handler,
17112 &incumbent_ctx,
17113 &mut incumbent_rx,
17114 hello_frame("aft", PROTOCOL_VERSION, 7),
17115 )
17116 .await;
17117 let (candidate_ctx, candidate_rx) = route_ctx(CANDIDATE);
17118 Swap {
17119 registry,
17120 forwarding,
17121 handler,
17122 incumbent_ctx,
17123 incumbent_rx,
17124 candidate_ctx,
17125 candidate_rx,
17126 }
17127 }
17128
17129 fn register_candidate(swap: &Swap, ready: Option<bool>) {
17130 let mut candidate_manifest = manifest("aft", PROTOCOL_VERSION);
17131 candidate_manifest.ready = ready;
17132 let registration = swap
17133 .registry
17134 .register_candidate_with_control_ops(
17135 candidate_manifest,
17136 PROTOCOL_VERSION,
17137 CANDIDATE,
17138 module_baseline_control_ops(),
17139 )
17140 .unwrap();
17141 swap.forwarding
17142 .register_candidate_module_connection(
17143 CANDIDATE,
17144 "aft".to_string(),
17145 PROTOCOL_VERSION,
17146 manifest_concurrency(®istration.manifest),
17147 swap.candidate_ctx.egress.clone(),
17148 )
17149 .unwrap();
17150 }
17151
17152 fn cutover(swap: &Swap) -> crate::forwarding::ModuleEndpointId {
17153 let cutover = swap.forwarding.cutover_candidate("aft").unwrap().unwrap();
17154 swap.registry.promote_candidate("aft").unwrap().unwrap();
17155 cutover.incumbent.unwrap()
17156 }
17157
17158 #[tokio::test]
17167 async fn swap_candidate_is_published_only_after_its_forwarding_endpoint_exists() {
17168 let mut swap = swap_with_incumbent().await;
17169 let hold = Arc::new(std::sync::Barrier::new(2));
17170 let handler = swap
17171 .handler
17172 .clone()
17173 .with_hello_registry_to_forwarding_hold(Arc::clone(&hold));
17174 handler
17175 .supervisor
17176 .open_swap("aft", "candidate-nonce".to_string());
17177 let hello = {
17178 let handler = handler.clone();
17179 let candidate_ctx = swap.candidate_ctx.clone();
17180 std::thread::spawn(move || {
17181 handler
17182 .handle_hello(
17183 candidate_ctx.connection_id,
17184 Some(candidate_ctx.egress.clone()),
17185 hello_frame_with_nonce(
17186 "aft",
17187 PROTOCOL_VERSION,
17188 8,
17189 Some("candidate-nonce"),
17190 ),
17191 )
17192 .unwrap()
17193 })
17194 };
17195
17196 hold.wait();
17197 assert!(swap
17200 .registry
17201 .registration(crate::registry::RegistrationSlot::Candidate("aft"))
17202 .unwrap()
17203 .is_some());
17204 assert!(swap
17205 .registry
17206 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
17207 .unwrap()
17208 .is_none());
17209 assert!(swap
17210 .forwarding
17211 .module_endpoint_for_connection(CANDIDATE)
17212 .unwrap()
17213 .is_none());
17214
17215 hold.wait();
17216 assert!(hello.join().unwrap().is_empty(), "candidate HELLO succeeds");
17217 assert_eq!(
17218 swap.candidate_rx.try_recv().unwrap().frame.header.ty,
17219 FrameType::HelloAck
17220 );
17221 let published = swap
17222 .registry
17223 .published_registration(crate::registry::RegistrationSlot::Candidate("aft"))
17224 .unwrap()
17225 .expect("candidate is published once its HELLO completes");
17226 assert_eq!(published.connection_id, CANDIDATE);
17227 assert!(swap
17228 .forwarding
17229 .module_endpoint_for_connection(CANDIDATE)
17230 .unwrap()
17231 .is_some());
17232 }
17233
17234 fn keyed_total(counters: &Value, key: &str) -> u64 {
17235 counters[key]
17236 .as_object()
17237 .map(|counts| counts.values().filter_map(Value::as_u64).sum())
17238 .unwrap_or(0)
17239 }
17240
17241 #[tokio::test]
17242 async fn replacement_logs_old_end_and_new_admission_once() {
17243 let (logs, _guard) = crate::router::test_log::log_capture(tracing::Level::INFO);
17244 let mut swap = swap_with_incumbent().await;
17245 swap.handler
17246 .supervisor
17247 .open_swap("aft", "candidate-nonce".to_string());
17248 hello_via_sink(
17249 &swap.handler,
17250 &swap.candidate_ctx,
17251 &mut swap.candidate_rx,
17252 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 8, Some("candidate-nonce")),
17253 )
17254 .await;
17255 cutover(&swap);
17256 swap.handler.cleanup_connection(INCUMBENT).unwrap();
17257
17258 let captured = crate::router::test_log::captured_logs(&logs);
17259 println!("captured replacement registry logs:\n{captured}");
17260 let new_admissions: Vec<_> = captured
17261 .lines()
17262 .filter(|line| {
17263 line.contains("connection_id=40")
17264 && line.contains("swap candidate registered; not routable until cutover")
17265 })
17266 .collect();
17267 assert_eq!(
17268 new_admissions.len(),
17269 1,
17270 "unexpected admission logs: {captured}"
17271 );
17272 assert!(new_admissions[0].contains("module_id=aft"));
17273 assert!(new_admissions[0].contains("connection_id=40"));
17274 assert!(
17275 new_admissions[0].contains("swap candidate registered; not routable until cutover")
17276 );
17277 assert!(new_admissions[0].contains("ready=true"));
17278
17279 let old_admissions: Vec<_> = captured
17280 .lines()
17281 .filter(|line| {
17282 line.contains("module registered module_id=aft ")
17283 && line.contains("connection_id=30")
17284 })
17285 .collect();
17286 assert_eq!(
17287 old_admissions.len(),
17288 1,
17289 "unexpected incumbent admission: {captured}"
17290 );
17291
17292 let old_ends: Vec<_> = captured
17293 .lines()
17294 .filter(|line| {
17295 line.contains("connection_id=30") && line.contains("module registration ended")
17296 })
17297 .collect();
17298 assert_eq!(old_ends.len(), 1, "unexpected end logs: {captured}");
17299 assert!(old_ends[0].contains("module_id=aft"));
17300 assert!(old_ends[0].contains("connection_id=30"));
17301 assert!(old_ends[0].contains("reason=replaced"));
17302 assert!(old_ends[0].contains("replaced_by_connection_id=40"));
17303 assert_eq!(
17304 captured
17305 .lines()
17306 .filter(|line| line.contains("module registration promoted"))
17307 .count(),
17308 1,
17309 "unexpected promotion logs: {captured}"
17310 );
17311 }
17312
17313 #[tokio::test]
17319 async fn incumbent_ack_between_promotion_and_drain_keeps_the_incumbent_serving() {
17320 let mut swap = swap_with_incumbent().await;
17321 let handler = swap.handler.clone();
17322
17323 let cotenant = ConnectionId::new(31);
17325 let (cotenant_ctx, mut cotenant_rx) = route_ctx(cotenant);
17326 let (cotenant_task, cotenant_bind) = relay_route_open(
17327 &handler,
17328 cotenant,
17329 &cotenant_ctx.egress,
17330 &mut swap.incumbent_rx,
17331 100,
17332 "aft",
17333 "swap-cotenant",
17334 )
17335 .await;
17336 handler
17337 .handle_control_frame(
17338 &swap.incumbent_ctx,
17339 route_bind_ack(cotenant_bind.header.corr),
17340 )
17341 .await
17342 .unwrap();
17343 assert!(cotenant_task.await.unwrap().is_empty());
17344 let (cotenant_channel, cotenant_epoch) =
17345 published_route(&cotenant_rx.recv().await.unwrap());
17346
17347 let caller = ConnectionId::new(32);
17349 let (caller_ctx, mut caller_rx) = route_ctx(caller);
17350 let (caller_task, caller_bind) = relay_route_open(
17351 &handler,
17352 caller,
17353 &caller_ctx.egress,
17354 &mut swap.incumbent_rx,
17355 101,
17356 "aft",
17357 "swap-caller",
17358 )
17359 .await;
17360 let (abandoned_channel, abandoned_epoch) = route_bind_channel(&caller_bind);
17361
17362 register_candidate(&swap, None);
17363 cutover(&swap);
17364
17365 let ack = handler
17367 .handle_control_frame(&swap.incumbent_ctx, route_bind_ack(caller_bind.header.corr))
17368 .await;
17369 let module_loop_error = ack.as_ref().err().map(ToString::to_string);
17370 if module_loop_error.is_some() {
17371 handler.cleanup_connection(INCUMBENT).unwrap();
17374 }
17375
17376 assert!(
17378 cotenant_rx.try_recv().is_err(),
17379 "the co-tenant route on the incumbent was torn down by one late ack: \
17380 {module_loop_error:?}"
17381 );
17382 assert!(matches!(
17383 swap.forwarding
17384 .lookup_data_route(cotenant, cotenant_channel, cotenant_epoch)
17385 .unwrap(),
17386 DataRoute::Client(DataRouteState::Bound(_))
17387 ));
17388 assert_eq!(module_loop_error, None);
17389 assert!(swap
17390 .registry
17391 .get_module_by_connection(INCUMBENT)
17392 .unwrap()
17393 .is_some());
17394
17395 let goodbye = tokio::time::timeout(Duration::from_secs(1), swap.incumbent_rx.recv())
17397 .await
17398 .expect("the incumbent is told to drop the abandoned binding")
17399 .unwrap()
17400 .frame;
17401 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
17402 assert_eq!(goodbye.header.channel, abandoned_channel);
17403 assert_eq!(goodbye.header.epoch, abandoned_epoch);
17404 assert!(swap.incumbent_rx.try_recv().is_err());
17405
17406 let response = caller_task.await.unwrap();
17408 assert_eq!(response.len(), 1);
17409 assert_eq!(parse_error(&response[0])["code"], "module_reloading");
17410 assert!(caller_rx.try_recv().is_err());
17411
17412 assert_eq!(swap.forwarding.reserved_route_count().unwrap(), (0, 0));
17416 let counters = handler.counters().snapshot();
17417 assert_eq!(
17418 keyed_total(&counters, "route_open_accepted_by_principal"),
17419 1
17420 );
17421 assert_eq!(keyed_total(&counters, "route_open_refused_by_code"), 1);
17422 assert_eq!(counters["route_open_refused_by_code"]["module_rejected"], 1);
17423 }
17424
17425 #[tokio::test]
17429 async fn route_open_after_cutover_and_incumbent_drain_is_relayed_to_the_candidate() {
17430 let mut swap = swap_with_incumbent().await;
17431 register_candidate(&swap, None);
17432 let incumbent = cutover(&swap);
17433 swap.forwarding
17434 .begin_endpoint_drain(incumbent, RouteCloseReason::Restart)
17435 .unwrap()
17436 .expect("the incumbent is still registered");
17437
17438 let client = ConnectionId::new(33);
17439 let (client_ctx, mut client_rx) = route_ctx(client);
17440 let route_handler = swap.handler.clone();
17441 let open_ctx = RouteCtx {
17442 connection_id: client,
17443 egress: client_ctx.egress.clone(),
17444 };
17445 let mut route_task = tokio::spawn(async move {
17446 route_handler
17447 .handle_control_frame(
17448 &open_ctx,
17449 route_open_frame(90, "aft", unique_project_root("swap-after-drain")),
17450 )
17451 .await
17452 .unwrap()
17453 });
17454 let bind = tokio::select! {
17455 bind = swap.candidate_rx.recv() => bind.expect("candidate egress is open").frame,
17456 response = &mut route_task => {
17457 let response = response.unwrap();
17458 panic!(
17459 "post-cutover route.open was refused instead of relayed to the candidate: {}",
17460 parse_error(&response[0])["code"]
17461 );
17462 }
17463 };
17464 swap.handler
17465 .handle_control_frame(&swap.candidate_ctx, route_bind_ack(bind.header.corr))
17466 .await
17467 .unwrap();
17468 assert!(route_task.await.unwrap().is_empty());
17469 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap());
17470 match swap
17471 .forwarding
17472 .lookup_data_route(client, channel, epoch)
17473 .unwrap()
17474 {
17475 DataRoute::Client(DataRouteState::Bound(route)) => {
17476 assert_eq!(route.module_endpoint.connection_id, CANDIDATE)
17477 }
17478 other => panic!("expected a bound route on the candidate, got {other:?}"),
17479 }
17480 assert!(swap.incumbent_rx.try_recv().is_err());
17481 }
17482
17483 #[tokio::test]
17488 async fn candidate_catalog_update_ready_reaches_the_candidate_registration() {
17489 let swap = swap_with_incumbent().await;
17490 register_candidate(&swap, Some(false));
17491 let update = Frame::build(
17492 FrameType::Request,
17493 control_flags(),
17494 0,
17495 0,
17496 55,
17497 serde_json::to_vec(&ModuleControlRequestFromModule::CatalogUpdate {
17498 provides: manifest("aft", PROTOCOL_VERSION).provides,
17499 capabilities: None,
17500 ready: Some(true),
17501 })
17502 .unwrap(),
17503 )
17504 .unwrap();
17505
17506 let replies = swap
17507 .handler
17508 .handle_control_frame(&swap.candidate_ctx, update)
17509 .await
17510 .unwrap();
17511
17512 assert_eq!(replies.len(), 1);
17513 assert_eq!(
17514 replies[0].header.ty,
17515 FrameType::Response,
17516 "candidate catalog.update was refused: {:?}",
17517 serde_json::from_slice::<Value>(&replies[0].body).ok()
17518 );
17519 assert!(swap.registry.get_candidate("aft").unwrap().unwrap().ready);
17520 assert_eq!(
17521 swap.registry
17522 .get_module("aft")
17523 .unwrap()
17524 .unwrap()
17525 .connection_id,
17526 INCUMBENT
17527 );
17528 }
17529 }
17530
17531 mod swap_admission {
17535 use super::*;
17536
17537 const INCUMBENT_NONCE: &str = "incumbent-nonce";
17538 const CANDIDATE_NONCE: &str = "candidate-nonce";
17539
17540 fn handler_with_incumbent(
17541 module_id: &str,
17542 reserved: bool,
17543 ) -> (Arc<Registry>, SupervisorHandle, ControlHandler) {
17544 let registry = Arc::new(Registry::default());
17545 let supervisor = SupervisorHandle::new();
17546 supervisor.set_spawn_nonce(module_id, INCUMBENT_NONCE.to_string());
17547 if reserved {
17548 supervisor.set_reserved_nonce(module_id, INCUMBENT_NONCE.to_string());
17549 }
17550 let handler =
17551 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor.clone());
17552 let incumbent = handler
17553 .handle_control(
17554 ConnectionId::new(1),
17555 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, 1, Some(INCUMBENT_NONCE)),
17556 )
17557 .unwrap();
17558 assert_eq!(incumbent[0].header.ty, FrameType::HelloAck);
17559 supervisor.open_swap(module_id, CANDIDATE_NONCE.to_string());
17560 (registry, supervisor, handler)
17561 }
17562
17563 #[test]
17569 fn unminted_nonce_on_an_unreserved_id_with_an_open_swap_is_refused() {
17570 let (registry, _supervisor, handler) = handler_with_incumbent("aft", false);
17571
17572 for (connection, nonce) in [(2, Some("forged")), (3, None)] {
17573 let replies = handler
17574 .handle_control(
17575 ConnectionId::new(connection),
17576 hello_frame_with_nonce("aft", PROTOCOL_VERSION, connection, nonce),
17577 )
17578 .unwrap();
17579 assert_eq!(replies[0].header.ty, FrameType::Error);
17580 assert_eq!(
17581 parse_error(&replies[0])["code"],
17582 "swap_token_invalid",
17583 "nonce {nonce:?}"
17584 );
17585 }
17586 assert!(registry.get_candidate("aft").unwrap().is_none());
17587 assert_eq!(
17588 registry.get_module("aft").unwrap().unwrap().connection_id,
17589 ConnectionId::new(1)
17590 );
17591
17592 let admitted = handler
17595 .handle_control(
17596 ConnectionId::new(4),
17597 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 4, Some(CANDIDATE_NONCE)),
17598 )
17599 .unwrap();
17600 assert_eq!(admitted[0].header.ty, FrameType::HelloAck);
17601 assert_eq!(
17602 registry
17603 .get_candidate("aft")
17604 .unwrap()
17605 .unwrap()
17606 .connection_id,
17607 ConnectionId::new(4)
17608 );
17609 assert_eq!(
17610 registry.get_module("aft").unwrap().unwrap().connection_id,
17611 ConnectionId::new(1),
17612 "the candidate must not take the active slot"
17613 );
17614 let replayed = handler
17615 .handle_control(
17616 ConnectionId::new(5),
17617 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 5, Some(CANDIDATE_NONCE)),
17618 )
17619 .unwrap();
17620 assert_eq!(parse_error(&replayed[0])["code"], "swap_token_invalid");
17621
17622 handler.cleanup_connection(ConnectionId::new(1)).unwrap();
17626 let squatter = handler
17627 .handle_control(
17628 ConnectionId::new(6),
17629 hello_frame_with_nonce("aft", PROTOCOL_VERSION, 6, Some("forged")),
17630 )
17631 .unwrap();
17632 assert_eq!(parse_error(&squatter[0])["code"], "swap_token_invalid");
17633 assert!(
17634 registry.get_module("aft").unwrap().is_none(),
17635 "a squatter took the active slot of an id being swapped"
17636 );
17637 }
17638
17639 #[test]
17644 fn reserved_module_candidate_is_admitted_ahead_of_the_reserved_gate() {
17645 let (registry, _supervisor, handler) = handler_with_incumbent("vault", true);
17646
17647 let replies = handler
17648 .handle_control(
17649 ConnectionId::new(2),
17650 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17651 )
17652 .unwrap();
17653
17654 assert_eq!(
17655 replies[0].header.ty,
17656 FrameType::HelloAck,
17657 "reserved candidate refused: {:?}",
17658 serde_json::from_slice::<Value>(&replies[0].body).ok()
17659 );
17660 assert_eq!(
17661 registry
17662 .get_candidate("vault")
17663 .unwrap()
17664 .unwrap()
17665 .connection_id,
17666 ConnectionId::new(2)
17667 );
17668 }
17669
17670 #[test]
17673 fn without_an_open_swap_the_ordinary_gates_decide() {
17674 let (registry, supervisor, handler) = handler_with_incumbent("vault", true);
17675 supervisor.close_swap("vault");
17676
17677 let candidate = handler
17678 .handle_control(
17679 ConnectionId::new(2),
17680 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 2, Some(CANDIDATE_NONCE)),
17681 )
17682 .unwrap();
17683 assert_eq!(parse_error(&candidate[0])["code"], "reserved_module");
17684 let duplicate = handler
17685 .handle_control(
17686 ConnectionId::new(3),
17687 hello_frame_with_nonce("vault", PROTOCOL_VERSION, 3, Some(INCUMBENT_NONCE)),
17688 )
17689 .unwrap();
17690 assert_eq!(parse_error(&duplicate[0])["code"], "duplicate_module_id");
17691 assert!(registry.get_candidate("vault").unwrap().is_none());
17692 }
17693 }
17694
17695 mod scopes {
17699 mod idle_tests {
17700 use super::*;
17701 include!("scope_idle_tests.rs");
17702 }
17703 mod route_open_timings {
17704 use super::*;
17705 include!("route_open_timing_tests.rs");
17706 }
17707 mod route_open_refusal_log {
17708 use super::*;
17709 include!("route_open_refusal_log_tests.rs");
17710 }
17711 mod activity {
17712 include!("activity_scope_tests.rs");
17713 }
17714 mod delta {
17715 include!("scope_handler_delta_tests.rs");
17716 }
17717 mod call_modes {
17718 include!("scope_call_mode_tests.rs");
17719 }
17720 mod federated {
17721 include!("scope_federated_tests.rs");
17722 }
17723 #[tokio::test]
17724 async fn served_by_dispatch_bypasses_exhaustive_decoder_and_requires_registration() {
17725 let rig = rig().await;
17726 let body = serde_json::to_vec(&subc_control::ScopeServedByRequest::new(
17727 "s".into(),
17728 1,
17729 "tool".into(),
17730 ))
17731 .unwrap();
17732 assert!(serde_json::from_slice::<ModuleControlRequestFromModule>(&body).is_err());
17733 let (direct, _rx) = wide_ctx(999);
17734 for (ctx, expected) in [
17735 (&rig.owner, FrameType::Response),
17736 (&direct, FrameType::Error),
17737 ] {
17738 let frame =
17739 Frame::build(FrameType::Request, control_flags(), 0, 0, 90, body.clone())
17740 .unwrap();
17741 let replies = rig.handler.handle_control_frame(ctx, frame).await.unwrap();
17742 assert_eq!(replies[0].header.ty, expected);
17743 if expected == FrameType::Response {
17744 let reply: subc_control::ScopeServedByReply =
17745 serde_json::from_slice(&replies[0].body).unwrap();
17746 assert_eq!(reply.answer, subc_control::ScopeServedByAnswer::Unknown);
17747 } else {
17748 assert_eq!(parse_error(&replies[0])["code"], "not_registered");
17749 }
17750 }
17751 let reply = call(
17752 &rig.handler,
17753 &rig.owner,
17754 &ModuleControlRequestFromModule::ScopeDescribe {
17755 owner: Principal::Reserved {
17756 module_id: OWNER.into(),
17757 },
17758 scope_ref: "s".into(),
17759 },
17760 )
17761 .await;
17762 assert!(matches!(
17763 serde_json::from_slice::<ModuleControlResponseToModule>(&reply.body).unwrap(),
17764 ModuleControlResponseToModule::ScopeDescribe { .. }
17765 ));
17766 }
17767
17768 use subc_protocol::scope::{
17769 ParentState, ScopeCarrier, ScopeKind, ScopeParent, ScopeRecordOutcome, ScopeStamp,
17770 ScopeStatus,
17771 };
17772
17773 use super::*;
17774
17775 const OWNER: &str = "prefrontal-core";
17776
17777 fn head(scope_ref: &str, scope_epoch: u64) -> ScopeRecord {
17778 ScopeRecord::new(scope_ref, scope_epoch, ScopeKind::Head)
17779 }
17780
17781 async fn call(
17782 handler: &ControlHandler,
17783 ctx: &RouteCtx,
17784 request: &ModuleControlRequestFromModule,
17785 ) -> Frame {
17786 let body = serde_json::to_vec(request).unwrap();
17787 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 77, body).unwrap();
17788 let mut replies = handler.handle_control_frame(ctx, frame).await.unwrap();
17789 assert_eq!(replies.len(), 1, "{replies:?}");
17790 replies.pop().unwrap()
17791 }
17792
17793 async fn sync(
17794 handler: &ControlHandler,
17795 ctx: &RouteCtx,
17796 generation: u64,
17797 scopes: Vec<ScopeRecord>,
17798 ) -> Result<ModuleControlResponseToModule, String> {
17799 let reply = call(
17800 handler,
17801 ctx,
17802 &ModuleControlRequestFromModule::ScopeSync { generation, scopes },
17803 )
17804 .await;
17805 match reply.header.ty {
17806 FrameType::Response => Ok(serde_json::from_slice(&reply.body).unwrap()),
17807 _ => Err(parse_error(&reply)["code"].as_str().unwrap().to_string()),
17808 }
17809 }
17810
17811 async fn describe(
17812 handler: &ControlHandler,
17813 ctx: &RouteCtx,
17814 owner: &str,
17815 scope_ref: &str,
17816 ) -> ModuleControlResponseToModule {
17817 let reply = call(
17818 handler,
17819 ctx,
17820 &ModuleControlRequestFromModule::ScopeDescribe {
17821 owner: Principal::Reserved {
17822 module_id: owner.to_string(),
17823 },
17824 scope_ref: scope_ref.to_string(),
17825 },
17826 )
17827 .await;
17828 assert_eq!(
17829 reply.header.ty,
17830 FrameType::Response,
17831 "{:?}",
17832 parse_error(&reply)
17833 );
17834 serde_json::from_slice(&reply.body).unwrap()
17835 }
17836
17837 #[tokio::test]
17838 async fn scope_describe_never_echoes_bind_proof_or_hash() {
17839 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
17840 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
17841 let mut rig = rig().await;
17842 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
17843 .await;
17844 let _bound = rig
17845 .bound(
17846 Some(AFT),
17847 PLEXUS,
17848 Some(
17849 rig.selector("s", Some(1))
17850 .with_bind_proof(Some(PROOF.into())),
17851 ),
17852 )
17853 .await;
17854 let reply = describe(&rig.handler, &rig.owner, OWNER, "s").await;
17855 let output = serde_json::to_string(&reply).unwrap();
17856 assert!(output.contains("\"status\":\"live\""));
17857 assert!(!output.contains(PROOF));
17858 assert!(!output.contains(HASH));
17859 }
17860
17861 async fn module(
17863 handler: &ControlHandler,
17864 connection: u64,
17865 module_id: &str,
17866 nonce: Option<&str>,
17867 ) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
17868 let (ctx, mut rx) = route_ctx(ConnectionId::new(connection));
17869 hello_via_sink(
17870 handler,
17871 &ctx,
17872 &mut rx,
17873 hello_frame_with_nonce(module_id, PROTOCOL_VERSION, connection, nonce),
17874 )
17875 .await;
17876 (ctx, rx)
17877 }
17878
17879 #[tokio::test]
17882 async fn a_client_connection_cannot_sync_or_describe() {
17883 let handler = ControlHandler::new(Arc::new(Registry::default()));
17884 let (ctx, _rx) = route_ctx(ConnectionId::new(9));
17885 for request in [
17886 ModuleControlRequestFromModule::ScopeSync {
17887 generation: 1,
17888 scopes: vec![head("s", 1)],
17889 },
17890 ModuleControlRequestFromModule::ScopeDescribe {
17891 owner: Principal::Direct,
17892 scope_ref: "s".to_string(),
17893 },
17894 ] {
17895 let reply = call(&handler, &ctx, &request).await;
17896 assert_eq!(parse_error(&reply)["code"], "not_registered", "{request:?}");
17897 }
17898 assert!(
17899 !handler
17900 .scopes
17901 .read()
17902 .unwrap()
17903 .describe(
17904 &Principal::Reserved {
17905 module_id: OWNER.to_string()
17906 },
17907 "s"
17908 )
17909 .owner_synced
17910 );
17911 }
17912
17913 #[tokio::test]
17916 async fn a_module_without_a_supervised_launch_cannot_sync() {
17917 let handler = ControlHandler::new(Arc::new(Registry::default()));
17918 let (ctx, _rx) = module(&handler, 1, OWNER, None).await;
17919 assert_eq!(
17920 sync(&handler, &ctx, 1, vec![head("s", 1)]).await,
17921 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17922 );
17923 }
17924
17925 #[tokio::test]
17926 async fn sync_authority_follows_the_supervisors_recorded_spawn_nonce_across_a_swap() {
17927 let supervisor = SupervisorHandle::new();
17928 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17929 let handler = ControlHandler::new(Arc::new(Registry::default()))
17930 .with_supervisor(supervisor.clone());
17931 let (incumbent, _incumbent_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17932 sync(&handler, &incumbent, 1, vec![head("s", 1)])
17933 .await
17934 .expect("the current launch syncs");
17935
17936 supervisor.open_swap(OWNER, "n2".to_string());
17939 let (candidate, _candidate_rx) = module(&handler, 2, OWNER, Some("n2")).await;
17940 assert_eq!(
17941 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17942 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17943 );
17944 sync(&handler, &incumbent, 2, vec![head("s", 1)])
17945 .await
17946 .expect("the serving owner syncs during the swap");
17947
17948 supervisor.close_swap(OWNER);
17951 assert_eq!(
17952 sync(&handler, &candidate, 1, vec![head("x", 1)]).await,
17953 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17954 );
17955 sync(&handler, &incumbent, 3, vec![head("s", 1)])
17956 .await
17957 .expect("the serving owner syncs after the rollback");
17958 handler.cleanup_connection(candidate.connection_id).unwrap();
17959
17960 supervisor.open_swap(OWNER, "n3".to_string());
17965 let (promoted, _promoted_rx) = module(&handler, 3, OWNER, Some("n3")).await;
17966 supervisor.set_spawn_nonce(OWNER, "n3".to_string());
17967 let reply = sync(&handler, &promoted, 1, vec![head("s", 1)])
17968 .await
17969 .expect("the promoted launch takes authority");
17970 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
17971 panic!("unexpected reply {reply:?}");
17972 };
17973 assert_eq!(results[0].outcome, ScopeRecordOutcome::Unchanged);
17974 assert_eq!(
17975 sync(&handler, &incumbent, 4, Vec::new()).await,
17976 Err(error_codes::SCOPE_SYNC_NOT_AUTHORITY.to_string())
17977 );
17978 }
17979
17980 #[tokio::test]
17983 async fn closing_the_authority_connection_frees_sync_authority() {
17984 let supervisor = SupervisorHandle::new();
17985 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
17986 let handler = ControlHandler::new(Arc::new(Registry::default()))
17987 .with_supervisor(supervisor.clone());
17988 let (first, _first_rx) = module(&handler, 1, OWNER, Some("n1")).await;
17989 sync(&handler, &first, 10, vec![head("s", 1)])
17990 .await
17991 .unwrap();
17992 handler.cleanup_connection(first.connection_id).unwrap();
17993
17994 let (second, _second_rx) = module(&handler, 2, OWNER, Some("n1")).await;
17995 sync(&handler, &second, 1, vec![head("s", 1)])
17996 .await
17997 .expect("the next connection takes the released authority");
17998 }
17999
18000 #[tokio::test]
18001 async fn module_goodbye_releases_scope_sync_authority_without_socket_close() {
18002 let supervisor = SupervisorHandle::new();
18003 supervisor.set_spawn_nonce(OWNER, "n1".to_string());
18004 let handler =
18005 ControlHandler::new(Arc::new(Registry::default())).with_supervisor(supervisor);
18006 let (first, _rx) = module(&handler, 1, OWNER, Some("n1")).await;
18007 sync(&handler, &first, 10, vec![head("s", 1)])
18008 .await
18009 .unwrap();
18010 handler
18011 .handle_control_frame(
18012 &first,
18013 Frame::build(FrameType::Goodbye, control_flags(), 0, 0, 4, Vec::new()).unwrap(),
18014 )
18015 .await
18016 .unwrap();
18017 let (second, _rx) = module(&handler, 2, OWNER, Some("n1")).await;
18018 sync(&handler, &second, 1, vec![head("s", 1)])
18019 .await
18020 .expect("GOODBYE releases authority even if the old socket remains open");
18021 }
18022
18023 #[tokio::test]
18024 async fn describe_reports_the_incarnation_and_whether_the_owner_is_configured() {
18025 let registry = Arc::new(Registry::default());
18026 let supervisor_handle = SupervisorHandle::new();
18027 let supervisor =
18028 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
18029 .with_handle(supervisor_handle.clone())
18030 .with_daemon_incarnation("incarnation-7".to_string());
18031 supervisor
18034 .supervise_configured(
18035 ModuleSpec {
18036 carry_containment: false,
18037 module_id: OWNER.to_string(),
18038 program: PathBuf::from("/nonexistent/prefrontal-core"),
18039 args: Vec::new(),
18040 env: Vec::new(),
18041 reserved: false,
18042 reserved_prefixes: Vec::new(),
18043 protocol: ModuleProtocol::Subc,
18044 overlap: Default::default(),
18045 },
18046 false,
18047 )
18048 .unwrap();
18049 supervisor_handle.set_spawn_nonce(OWNER, "n1".to_string());
18050 let handler =
18051 ControlHandler::new(Arc::clone(®istry)).with_supervisor(supervisor_handle);
18052 let (reader, _reader_rx) = module(&handler, 5, "reader", None).await;
18053
18054 let ModuleControlResponseToModule::ScopeDescribe {
18056 status,
18057 daemon_incarnation,
18058 owner_synced,
18059 owner_configured,
18060 scope,
18061 ..
18062 } = describe(&handler, &reader, OWNER, "s").await
18063 else {
18064 panic!("not a describe reply");
18065 };
18066 assert_eq!(status, ScopeStatus::NotLive);
18067 assert_eq!(daemon_incarnation, "incarnation-7");
18068 assert!(!owner_synced);
18069 assert!(owner_configured);
18070 assert!(scope.is_none());
18071
18072 let ModuleControlResponseToModule::ScopeDescribe {
18075 status,
18076 owner_configured,
18077 ..
18078 } = describe(&handler, &reader, "ghost", "s").await
18079 else {
18080 panic!("not a describe reply");
18081 };
18082 assert_eq!(status, ScopeStatus::NotLive);
18083 assert!(!owner_configured);
18084
18085 let (owner, _owner_rx) = module(&handler, 6, OWNER, Some("n1")).await;
18087 sync(&handler, &owner, 1, vec![head("s", 4)]).await.unwrap();
18088 let ModuleControlResponseToModule::ScopeDescribe {
18089 status,
18090 scope_epoch,
18091 owner_synced,
18092 scope,
18093 ..
18094 } = describe(&handler, &reader, OWNER, "s").await
18095 else {
18096 panic!("not a describe reply");
18097 };
18098 assert_eq!(status, ScopeStatus::Live);
18099 assert_eq!(scope_epoch, Some(4));
18100 assert!(owner_synced);
18101 let stamp = scope.expect("a live scope carries its stamp");
18102 assert!(
18103 stamp.owner_authorized,
18104 "prefrontal-core is the default authority"
18105 );
18106 assert_eq!(stamp.kind, ScopeKind::Head);
18107 }
18108
18109 #[tokio::test]
18110 async fn scope_authority_owners_decides_owner_authorized() {
18111 let supervisor = SupervisorHandle::new();
18112 supervisor.set_spawn_nonce("broca", "b1".to_string());
18113 let handler = ControlHandler::new(Arc::new(Registry::default()))
18114 .with_supervisor(supervisor)
18115 .with_scope_authority_owners(vec!["broca".to_string()]);
18116 let (broca, _rx) = module(&handler, 1, "broca", Some("b1")).await;
18117 let mut gated = head("s", 1);
18118 gated.attributes.agent_id = Some("agent".to_string());
18119 sync(&handler, &broca, 1, vec![gated]).await.unwrap();
18120 let ModuleControlResponseToModule::ScopeDescribe { scope, .. } =
18121 describe(&handler, &broca, "broca", "s").await
18122 else {
18123 panic!("not a describe reply");
18124 };
18125 assert!(scope.unwrap().owner_authorized);
18126 }
18127
18128 #[tokio::test]
18132 async fn scope_ops_and_the_scopes_capability_are_advertised() {
18133 let handler = ControlHandler::new(Arc::new(Registry::default()));
18134 let (ctx, mut rx) = route_ctx(ConnectionId::new(1));
18135 let ack = hello_via_sink(
18136 &handler,
18137 &ctx,
18138 &mut rx,
18139 hello_frame("m", PROTOCOL_VERSION, 1),
18140 )
18141 .await;
18142 let ack = parse_ack(&ack);
18143 for op in [SCOPE_SYNC_OP, SCOPE_DESCRIBE_OP] {
18144 assert!(ack.subc_ops.iter().any(|o| o == op), "{:?}", ack.subc_ops);
18145 }
18146 assert!(ack.subc_capabilities.iter().any(|c| c == CAP_SCOPES_V1));
18147
18148 let (client, _client_rx) = route_ctx(ConnectionId::new(2));
18149 let body = serde_json::to_vec(&ClientControlRequest::ServerDescribe {}).unwrap();
18150 let frame = Frame::build(FrameType::Request, control_flags(), 0, 0, 5, body).unwrap();
18151 let reply = handler
18152 .handle_control_frame(&client, frame)
18153 .await
18154 .unwrap()
18155 .pop()
18156 .unwrap();
18157 let ClientControlResponse::ServerDescribe { capabilities, .. } =
18158 serde_json::from_slice(&reply.body).unwrap()
18159 else {
18160 panic!("not a server.describe reply");
18161 };
18162 assert!(
18163 capabilities.iter().any(|c| c == CAP_SCOPES_V1),
18164 "{capabilities:?}"
18165 );
18166 }
18167
18168 const PLEXUS: &str = "plexus";
18171 const OTHER: &str = "other";
18172 const AFT: &str = "aft";
18173 const BROCA: &str = "broca";
18174 const MAGIC: &str = "magic-context";
18175
18176 fn nonce(module_id: &str) -> String {
18177 format!("nonce-{module_id}")
18178 }
18179
18180 fn wide_ctx(connection: u64) -> (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>) {
18181 let (tx, rx) = mpsc::channel(64);
18182 (
18183 RouteCtx {
18184 connection_id: ConnectionId::new(connection),
18185 egress: FrameSink::new(tx),
18186 },
18187 rx,
18188 )
18189 }
18190
18191 struct Rig {
18195 handler: ControlHandler,
18196 forwarding: Arc<ForwardingTable>,
18197 owner: RouteCtx,
18198 _owner_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18199 modules: BTreeMap<String, (RouteCtx, mpsc::Receiver<crate::router::OutboundFrame>)>,
18200 generation: u64,
18201 next_connection: u64,
18202 _supervisor: Supervisor,
18203 }
18204
18205 async fn rig() -> Rig {
18206 rig_with_flow_support(true).await
18207 }
18208
18209 async fn rig_with_flow_support(flow_support: bool) -> Rig {
18210 rig_with_scope_capabilities(flow_support, false).await
18211 }
18212
18213 async fn rig_with_scope_capabilities(flow_support: bool, run_support: bool) -> Rig {
18214 let mut provides = Vec::new();
18215 if flow_support {
18216 provides.push("flow-scopes/v1");
18217 }
18218 if run_support {
18219 provides.push("agent-run-scopes/v1");
18220 }
18221 rig_with_target_capabilities(&provides).await
18222 }
18223
18224 fn hello_with_capabilities(module_id: &str, connection: u64, provides: &[&str]) -> Frame {
18225 let hello = hello_frame(module_id, PROTOCOL_VERSION, connection);
18226 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18227 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18230 body["manifest"]["capabilities"] = serde_json::json!({"provides": provides});
18231 Frame::build(
18232 FrameType::Hello,
18233 control_flags(),
18234 0,
18235 0,
18236 connection,
18237 serde_json::to_vec(&body).unwrap(),
18238 )
18239 .unwrap()
18240 }
18241
18242 async fn rig_with_target_capabilities(provides: &[&str]) -> Rig {
18243 let registry = Arc::new(Registry::default());
18244 let forwarding = Arc::new(ForwardingTable::default());
18245 let supervisor_handle = SupervisorHandle::new();
18246 let supervisor =
18247 Supervisor::new_for_test(Arc::clone(®istry), RestartPolicy::default())
18248 .with_handle(supervisor_handle.clone());
18249 supervisor
18250 .supervise_configured(
18251 ModuleSpec {
18252 carry_containment: false,
18253 module_id: OWNER.to_string(),
18254 program: PathBuf::from("/nonexistent/prefrontal-core"),
18255 args: Vec::new(),
18256 env: Vec::new(),
18257 reserved: false,
18258 reserved_prefixes: Vec::new(),
18259 protocol: ModuleProtocol::Subc,
18260 overlap: Default::default(),
18261 },
18262 false,
18263 )
18264 .unwrap();
18265 for module_id in [OWNER, AFT, BROCA, MAGIC] {
18266 supervisor_handle.set_spawn_nonce(module_id, nonce(module_id));
18267 }
18268 let handler =
18269 ControlHandler::with_forwarding(Arc::clone(®istry), Arc::clone(&forwarding))
18270 .with_supervisor(supervisor_handle);
18271 let (owner, mut owner_rx) = wide_ctx(1);
18272 hello_via_sink(
18273 &handler,
18274 &owner,
18275 &mut owner_rx,
18276 hello_frame_with_nonce(OWNER, PROTOCOL_VERSION, 1, Some(&nonce(OWNER))),
18277 )
18278 .await;
18279 let mut modules = BTreeMap::new();
18280 for (connection, module_id) in [(2, PLEXUS), (3, OTHER)] {
18281 let (ctx, mut rx) = wide_ctx(connection);
18282 let hello = hello_with_capabilities(module_id, connection, provides);
18283 hello_via_sink(&handler, &ctx, &mut rx, hello).await;
18284 modules.insert(module_id.to_string(), (ctx, rx));
18285 }
18286 Rig {
18287 handler,
18288 forwarding,
18289 owner,
18290 _owner_rx: owner_rx,
18291 modules,
18292 generation: 0,
18293 next_connection: 100,
18294 _supervisor: supervisor,
18295 }
18296 }
18297
18298 fn carrier(module_id: &str, targets: Option<&[&str]>) -> ScopeCarrier {
18299 ScopeCarrier::new(Principal::Reserved {
18300 module_id: module_id.to_string(),
18301 })
18302 .with_targets(targets.map(|targets| targets.iter().map(|t| t.to_string()).collect()))
18303 }
18304
18305 fn session(scope_epoch: u64) -> ScopeRecord {
18308 let mut record = head("s", scope_epoch);
18309 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS, OTHER]))];
18310 record.attributes.agent_id = Some("agent-1".to_string());
18311 record.attributes.delegates = true;
18312 record
18313 }
18314
18315 impl Rig {
18316 async fn sync(&mut self, scopes: Vec<ScopeRecord>) {
18317 self.generation += 1;
18318 sync(&self.handler, &self.owner, self.generation, scopes)
18319 .await
18320 .expect("the owner's sync is accepted");
18321 }
18322
18323 fn selector(&self, scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18324 ScopeSelector::new(
18325 Principal::Reserved {
18326 module_id: OWNER.to_string(),
18327 },
18328 scope_ref.to_string(),
18329 scope_epoch,
18330 )
18331 }
18332
18333 fn open_frame(
18334 &mut self,
18335 opener: Option<&str>,
18336 target: &str,
18337 scope: Option<ScopeSelector>,
18338 ) -> (
18339 RouteCtx,
18340 mpsc::Receiver<crate::router::OutboundFrame>,
18341 Frame,
18342 ) {
18343 self.next_connection += 1;
18344 let (ctx, rx) = wide_ctx(self.next_connection);
18345 let body = serde_json::to_vec(&ClientControlRequest::RouteOpen {
18346 busy_refusals: None,
18347 target: RouteTarget::ToolProvider {
18348 module_id: target.to_string(),
18349 },
18350 identity: BindIdentity::new("unit".to_string(), "session".to_string()),
18351 consumer_identity: opener.map(|module_id| ConsumerIdentity {
18352 module_id: module_id.to_string(),
18353 launch_nonce: nonce(module_id),
18354 }),
18355 consumer_capabilities: None,
18356 role_versions: None,
18357 admission_facts: None,
18358 scope,
18359 })
18360 .unwrap();
18361 let frame = Frame::build(
18362 FrameType::Request,
18363 control_flags(),
18364 0,
18365 0,
18366 self.next_connection,
18367 body,
18368 )
18369 .unwrap();
18370 (ctx, rx, frame)
18371 }
18372
18373 async fn refused(
18375 &mut self,
18376 opener: Option<&str>,
18377 target: &str,
18378 scope: Option<ScopeSelector>,
18379 ) -> String {
18380 self.refusal_body(opener, target, scope).await["code"]
18381 .as_str()
18382 .unwrap()
18383 .to_string()
18384 }
18385
18386 async fn refusal_body(
18387 &mut self,
18388 opener: Option<&str>,
18389 target: &str,
18390 scope: Option<ScopeSelector>,
18391 ) -> Value {
18392 let (ctx, _rx, frame) = self.open_frame(opener, target, scope);
18393 let replies = tokio::time::timeout(
18394 Duration::from_secs(2),
18395 self.handler.handle_control_frame(&ctx, frame),
18396 )
18397 .await
18398 .expect("the open must be refused before waiting for a bind ack")
18399 .unwrap();
18400 assert_eq!(replies.len(), 1, "{replies:?}");
18401 assert_eq!(replies[0].header.ty, FrameType::Error);
18402 let (_, module_rx) = self.modules.get_mut(target).unwrap();
18403 assert!(
18404 module_rx.try_recv().is_err(),
18405 "a refused open relays nothing"
18406 );
18407 assert_eq!(self.forwarding.reserved_route_count().unwrap(), (0, 0));
18408 parse_error(&replies[0])
18409 }
18410
18411 async fn relayed(
18413 &mut self,
18414 opener: Option<&str>,
18415 target: &str,
18416 scope: Option<ScopeSelector>,
18417 ) -> Relayed {
18418 self.relayed_with_legacy_root(opener, target, scope, None)
18419 .await
18420 }
18421
18422 async fn relayed_with_legacy_root(
18423 &mut self,
18424 opener: Option<&str>,
18425 target: &str,
18426 scope: Option<ScopeSelector>,
18427 legacy_root: Option<&std::path::Path>,
18428 ) -> Relayed {
18429 let (ctx, rx, mut frame) = self.open_frame(opener, target, scope);
18430 if let Some(root) = legacy_root {
18431 let mut body: Value = serde_json::from_slice(&frame.body).unwrap();
18432 body["identity"]["project_root"] = serde_json::json!(root);
18433 frame.body = serde_json::to_vec(&body).unwrap();
18434 }
18435 let handler = self.handler.clone();
18436 let task_ctx = ctx.clone();
18437 let task = tokio::spawn(async move {
18438 handler
18439 .handle_control_frame(&task_ctx, frame)
18440 .await
18441 .unwrap()
18442 });
18443 let (_, module_rx) = self.modules.get_mut(target).unwrap();
18444 let bind = tokio::time::timeout(Duration::from_secs(2), module_rx.recv())
18445 .await
18446 .expect("the target receives the relayed route.bind")
18447 .unwrap()
18448 .frame;
18449 Relayed {
18450 target: target.to_string(),
18451 client: ctx,
18452 client_rx: rx,
18453 task,
18454 bind,
18455 }
18456 }
18457
18458 async fn ack(&self, relayed: &Relayed) {
18459 let (module, _) = &self.modules[&relayed.target];
18460 self.handler
18461 .handle_control_frame(module, route_bind_ack(relayed.bind.header.corr))
18462 .await
18463 .unwrap();
18464 }
18465
18466 async fn bound(
18468 &mut self,
18469 opener: Option<&str>,
18470 target: &str,
18471 scope: Option<ScopeSelector>,
18472 ) -> Bound {
18473 let relayed = self.relayed(opener, target, scope).await;
18474 self.commit_relayed(relayed).await
18475 }
18476
18477 async fn bound_with_legacy_root(
18478 &mut self,
18479 opener: Option<&str>,
18480 target: &str,
18481 scope: Option<ScopeSelector>,
18482 root: &std::path::Path,
18483 ) -> Bound {
18484 let relayed = self
18485 .relayed_with_legacy_root(opener, target, scope, Some(root))
18486 .await;
18487 self.commit_relayed(relayed).await
18488 }
18489
18490 async fn commit_relayed(&self, relayed: Relayed) -> Bound {
18491 self.ack(&relayed).await;
18492 let Relayed {
18493 target,
18494 client,
18495 mut client_rx,
18496 task,
18497 bind,
18498 } = relayed;
18499 assert!(
18500 task.await.unwrap().is_empty(),
18501 "the open is answered by commit"
18502 );
18503 let (channel, epoch) = published_route(&client_rx.recv().await.unwrap().frame);
18504 Bound {
18505 target,
18506 client,
18507 client_rx,
18508 channel,
18509 epoch,
18510 bind,
18511 }
18512 }
18513
18514 fn live(&self, route: &Bound) -> bool {
18515 matches!(
18516 self.forwarding
18517 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
18518 .unwrap(),
18519 DataRoute::Client(DataRouteState::Bound(_))
18520 )
18521 }
18522 }
18523
18524 struct Relayed {
18525 target: String,
18526 client: RouteCtx,
18527 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18528 task: tokio::task::JoinHandle<Vec<Frame>>,
18529 bind: Frame,
18530 }
18531
18532 struct Bound {
18533 target: String,
18534 client: RouteCtx,
18535 client_rx: mpsc::Receiver<crate::router::OutboundFrame>,
18536 channel: u16,
18537 epoch: u32,
18538 bind: Frame,
18539 }
18540
18541 impl Bound {
18542 fn closed_reason(&mut self) -> RouteCloseReason {
18545 let mut reason = None;
18546 let mut goodbye = false;
18547 while let Ok(outbound) = self.client_rx.try_recv() {
18548 let frame = outbound.frame;
18549 match frame.header.ty {
18550 FrameType::Goodbye => {
18551 assert_eq!(
18552 (frame.header.channel, frame.header.epoch),
18553 (self.channel, self.epoch)
18554 );
18555 goodbye = true;
18556 }
18557 FrameType::Push => {
18558 let ClientControlPush::RouteClosed {
18559 reason: r,
18560 module_id,
18561 ..
18562 } = serde_json::from_slice(&frame.body).unwrap()
18563 else {
18564 panic!("unexpected push");
18565 };
18566 assert_eq!(module_id, self.target);
18567 reason = Some(r);
18568 }
18569 other => panic!("unexpected frame {other:?}"),
18570 }
18571 }
18572 assert!(goodbye, "the client is sent a GOODBYE for the closed route");
18573 reason.expect("the client is told why the route closed")
18574 }
18575
18576 fn untouched(&mut self) -> bool {
18577 self.client_rx.try_recv().is_err()
18578 }
18579
18580 fn stamp(&self) -> Option<ScopeStamp> {
18581 match serde_json::from_slice::<ModuleControlRequest>(&self.bind.body).unwrap() {
18582 ModuleControlRequest::RouteBind { scope, .. } => scope,
18583 other => panic!("expected a route.bind, got {other:?}"),
18584 }
18585 }
18586 }
18587
18588 #[tokio::test]
18589 async fn only_the_owner_or_a_listed_carrier_is_admitted_and_a_targeted_carrier_only_to_its_modules(
18590 ) {
18591 let mut rig = rig().await;
18592 let mut record = session(1);
18593 record.carriers = vec![carrier(AFT, None), carrier(BROCA, Some(&[PLEXUS]))];
18594 record.child_owners = vec![Principal::Reserved {
18595 module_id: MAGIC.to_string(),
18596 }];
18597 rig.sync(vec![record]).await;
18598 let scope = || Some(rig_selector("s", Some(1)));
18599
18600 rig.bound(Some(OWNER), PLEXUS, scope()).await;
18603 rig.bound(Some(AFT), OTHER, scope()).await;
18604 rig.bound(Some(BROCA), PLEXUS, scope()).await;
18605
18606 for (opener, target) in [(Some(BROCA), OTHER), (Some(MAGIC), PLEXUS), (None, PLEXUS)] {
18610 assert_eq!(
18611 rig.refused(opener, target, scope()).await,
18612 error_codes::SCOPE_NOT_CARRIER,
18613 "{opener:?} -> {target}"
18614 );
18615 }
18616 }
18617
18618 fn rig_selector(scope_ref: &str, scope_epoch: Option<u64>) -> ScopeSelector {
18619 ScopeSelector::new(
18620 Principal::Reserved {
18621 module_id: OWNER.to_string(),
18622 },
18623 scope_ref.to_string(),
18624 scope_epoch,
18625 )
18626 }
18627
18628 #[tokio::test]
18629 async fn an_open_without_an_epoch_is_refused_the_owners_included() {
18630 let mut rig = rig().await;
18631 rig.sync(vec![session(1)]).await;
18632 for opener in [OWNER, AFT] {
18633 assert_eq!(
18634 rig.refused(Some(opener), PLEXUS, Some(rig.selector("s", None)))
18635 .await,
18636 error_codes::SCOPE_EPOCH_REQUIRED,
18637 "{opener}"
18638 );
18639 }
18640 }
18641
18642 #[tokio::test]
18643 async fn admission_separates_not_synced_not_live_and_ended() {
18644 let mut rig = rig().await;
18645 let code = rig
18647 .refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18648 .await;
18649 assert_eq!(code, error_codes::SCOPE_NOT_SYNCED);
18650 assert!(subc_protocol::error_codes::is_retryable_route_open(&code));
18651
18652 let ghost = ScopeSelector::new(
18654 Principal::Reserved {
18655 module_id: "ghost".to_string(),
18656 },
18657 "s".to_string(),
18658 Some(1),
18659 );
18660 assert_eq!(
18661 rig.refused(Some(AFT), PLEXUS, Some(ghost)).await,
18662 error_codes::SCOPE_NOT_LIVE
18663 );
18664
18665 rig.sync(vec![session(2)]).await;
18666 assert_eq!(
18667 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("missing", Some(1))))
18668 .await,
18669 error_codes::SCOPE_NOT_LIVE
18670 );
18671 for epoch in [1, 3] {
18672 assert_eq!(
18673 rig.refused(Some(AFT), PLEXUS, Some(rig_selector("s", Some(epoch))))
18674 .await,
18675 error_codes::SCOPE_ENDED,
18676 "epoch {epoch}"
18677 );
18678 }
18679 rig.bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(2))))
18681 .await;
18682 }
18683
18684 #[tokio::test]
18685 async fn the_bind_is_stamped_and_owner_authorized_only_for_listed_owners() {
18686 let mut rig = rig().await;
18687 rig.sync(vec![session(1)]).await;
18688 let route = rig
18689 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18690 .await;
18691 let stamp = route.stamp().expect("a scoped bind carries the stamp");
18692 assert_eq!(stamp.scope_ref, "s");
18693 assert_eq!(stamp.scope_epoch, 1);
18694 assert_eq!(stamp.kind, ScopeKind::Head);
18695 assert_eq!(stamp.attributes.agent_id.as_deref(), Some("agent-1"));
18696 assert!(stamp.attributes.delegates);
18697 assert!(stamp.owner_authorized);
18698 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18699 assert_eq!(unscoped.stamp(), None, "an unscoped open is not stamped");
18700
18701 let (broca, mut broca_rx) = wide_ctx(50);
18704 hello_via_sink(
18705 &rig.handler,
18706 &broca,
18707 &mut broca_rx,
18708 hello_frame_with_nonce(BROCA, PROTOCOL_VERSION, 50, Some(&nonce(BROCA))),
18709 )
18710 .await;
18711 sync(&rig.handler, &broca, 1, vec![head("b", 1)])
18712 .await
18713 .unwrap();
18714 let own = ScopeSelector::new(
18715 Principal::Reserved {
18716 module_id: BROCA.to_string(),
18717 },
18718 "b".to_string(),
18719 Some(1),
18720 );
18721 let route = rig.bound(Some(BROCA), PLEXUS, Some(own)).await;
18722 assert!(!route.stamp().unwrap().owner_authorized);
18723 }
18724
18725 #[tokio::test]
18726 async fn an_authority_owners_flow_id_without_an_agent_is_stamped_verbatim_on_bind() {
18727 let mut rig = rig().await;
18728 let mut record = head("s", 1);
18729 record.carriers = vec![carrier(AFT, None)];
18730 let flow_id = "Flow:run-7/step_2!~";
18731 record.attributes.flow_id = Some(flow_id.to_string());
18732 let reply = sync(&rig.handler, &rig.owner, 1, vec![record])
18733 .await
18734 .unwrap();
18735 let ModuleControlResponseToModule::ScopeSync { results, .. } = reply else {
18736 panic!("not a sync reply");
18737 };
18738 assert_eq!(results[0].outcome, ScopeRecordOutcome::Created);
18739 let route = rig
18740 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18741 .await;
18742 assert!(rig.live(&route), "the stamped bind committed");
18743 let stamp = route.stamp().expect("a flow scope carries a stamp");
18744 assert_eq!(stamp.attributes.flow_id.as_deref(), Some(flow_id));
18745 assert_eq!(stamp.attributes.agent_id, None);
18746 assert!(!stamp.attributes.delegates);
18747 assert!(stamp.owner_authorized);
18748 let unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
18749 assert_eq!(unscoped.stamp(), None);
18750 }
18751
18752 #[tokio::test]
18753 async fn flow_scope_refuses_a_0_29_target_without_flow_capability_and_relays_nothing() {
18754 let mut rig = rig_with_flow_support(false).await;
18755 let mut record = session(1);
18756 record.attributes.flow_id = Some("flow:7".to_string());
18757 rig.sync(vec![record]).await;
18758 for opener in [OWNER, AFT] {
18759 let body = rig
18760 .refusal_body(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18761 .await;
18762 assert_eq!(body["code"], "target_flow_unsupported");
18763 let message = body["message"].as_str().unwrap();
18764 for required in [PLEXUS, "flow-scopes/v1"] {
18765 assert!(message.contains(required), "{message}");
18766 }
18767 }
18768 }
18769
18770 #[tokio::test]
18771 async fn flow_scope_admits_a_capable_target_and_preserves_flow_id_on_bind() {
18772 let mut rig = rig_with_flow_support(true).await;
18773 let mut record = session(1);
18774 record.attributes.flow_id = Some("flow:7".to_string());
18775 rig.sync(vec![record]).await;
18776 for opener in [OWNER, AFT] {
18777 let route = rig
18778 .bound(Some(opener), PLEXUS, Some(rig_selector("s", Some(1))))
18779 .await;
18780 assert!(rig.live(&route));
18781 assert_eq!(
18782 route.stamp().unwrap().attributes.flow_id.as_deref(),
18783 Some("flow:7")
18784 );
18785 }
18786 }
18787
18788 #[tokio::test]
18789 async fn flow_scope_rechecks_the_relay_target_after_a_reconnect() {
18790 use std::future::Future;
18791
18792 let mut rig = rig_with_flow_support(true).await;
18793 let mut record = session(1);
18794 record.attributes.flow_id = Some("flow:7".to_string());
18795 rig.sync(vec![record]).await;
18796 let (client, mut client_rx, frame) =
18797 rig.open_frame(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))));
18798 for _ in 0..64 {
18801 client.egress.try_send(route_bind_ack(1)).unwrap();
18802 }
18803 let handler = rig.handler.clone();
18804 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18805 std::future::poll_fn(|cx| {
18806 assert!(open.as_mut().poll(cx).is_pending());
18807 std::task::Poll::Ready(())
18808 })
18809 .await;
18810 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18811
18812 let old_connection = rig.modules[PLEXUS].0.connection_id;
18813 rig.handler.cleanup_connection(old_connection).unwrap();
18814 let (replacement, mut replacement_rx) = wide_ctx(200);
18815 let hello = hello_frame(PLEXUS, PROTOCOL_VERSION, 200);
18816 let mut body: Value = serde_json::from_slice(&hello.body).unwrap();
18817 body["manifest"]["provenance"] = serde_json::json!({"wire_crate_version": "0.29.0"});
18818 let hello = Frame::build(
18819 FrameType::Hello,
18820 control_flags(),
18821 0,
18822 0,
18823 200,
18824 serde_json::to_vec(&body).unwrap(),
18825 )
18826 .unwrap();
18827 hello_via_sink(&rig.handler, &replacement, &mut replacement_rx, hello).await;
18828
18829 client_rx.try_recv().unwrap();
18830 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18831 .await
18832 .expect("the replacement is refused without waiting for a bind ack")
18833 .unwrap();
18834 assert_eq!(replies.len(), 1);
18835 let body = parse_error(&replies[0]);
18836 assert_eq!(body["code"], "target_flow_unsupported");
18837 for required in [PLEXUS, "flow-scopes/v1"] {
18838 assert!(body["message"].as_str().unwrap().contains(required));
18839 }
18840 assert!(replacement_rx.try_recv().is_err(), "no bind is relayed");
18841 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18842 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18843 assert!(rig
18844 .handler
18845 .registry
18846 .get_module_by_connection(replacement.connection_id)
18847 .unwrap()
18848 .is_some());
18849 }
18850
18851 #[tokio::test]
18852 async fn scope_bind_proof_rechecks_current_hash_before_captured_relay() {
18853 use std::future::Future;
18854 const PROOF: &str = "BwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwcHBwc";
18855 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18856 const NEXT_HASH: &str =
18857 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18858 let mut rig = rig().await;
18859 let mut record = session(1).with_bind_proof_sha256(Some(HASH.into()));
18860 record.carriers.push(ScopeCarrier::new(Principal::Direct));
18861 rig.sync(vec![record.clone()]).await;
18862 let (client, mut rx, frame) = rig.open_frame(
18863 None,
18864 PLEXUS,
18865 Some(
18866 rig.selector("s", Some(1))
18867 .with_bind_proof(Some(PROOF.into())),
18868 ),
18869 );
18870 for _ in 0..64 {
18873 client.egress.try_send(route_bind_ack(1)).unwrap();
18874 }
18875 let handler = rig.handler.clone();
18876 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18877 std::future::poll_fn(|cx| {
18878 assert!(open.as_mut().poll(cx).is_pending());
18879 std::task::Poll::Ready(())
18880 })
18881 .await;
18882 record.bind_proof_sha256 = Some(NEXT_HASH.into());
18883 rig.sync(vec![record]).await;
18884 rx.try_recv().unwrap();
18885 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18886 .await
18887 .unwrap()
18888 .unwrap();
18889 assert_eq!(
18890 parse_error(&replies[0])["code"],
18891 error_codes::SCOPE_BIND_PROOF_MISMATCH
18892 );
18893 assert!(!String::from_utf8_lossy(&replies[0].body).contains(PROOF));
18894 assert!(rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_err());
18895 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18896 }
18897
18898 #[tokio::test]
18899 async fn scope_bind_proof_reserved_openers_ignore_proof() {
18900 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18901 let mut rig = rig().await;
18902 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18903 .await;
18904 for opener in [OWNER, AFT] {
18905 for proof in [None, Some("malformed+proof=")] {
18906 let route = rig
18907 .bound(
18908 Some(opener),
18909 PLEXUS,
18910 Some(
18911 rig.selector("s", Some(1))
18912 .with_bind_proof(proof.map(str::to_string)),
18913 ),
18914 )
18915 .await;
18916 assert!(rig.live(&route));
18917 }
18918 }
18919 }
18920
18921 #[tokio::test]
18922 async fn scope_bind_proof_reserved_opener_passes_hash_change_before_captured_relay() {
18923 use std::future::Future;
18924 const HASH: &str = "4bb06f8e4e3a7715d201d573d0aa423762e55dabd61a2c02278fa56cc6d294e0";
18925 const NEXT_HASH: &str =
18926 "2578ccf8645b2d1dc10c465eff843585970f3a7e22296a92cad55d489a272072";
18927 let mut rig = rig().await;
18928 rig.sync(vec![session(1).with_bind_proof_sha256(Some(HASH.into()))])
18929 .await;
18930 let (client, mut rx, frame) =
18931 rig.open_frame(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))));
18932 for _ in 0..64 {
18933 client.egress.try_send(route_bind_ack(1)).unwrap();
18934 }
18935 let handler = rig.handler.clone();
18936 let mut open = Box::pin(handler.handle_control_frame(&client, frame));
18937 std::future::poll_fn(|cx| {
18938 assert!(open.as_mut().poll(cx).is_pending());
18939 std::task::Poll::Ready(())
18940 })
18941 .await;
18942 rig.sync(vec![
18943 session(1).with_bind_proof_sha256(Some(NEXT_HASH.into()))
18944 ])
18945 .await;
18946 rx.try_recv().unwrap();
18947 let (_, module_rx) = rig.modules.get_mut(PLEXUS).unwrap();
18948 let bind = std::future::poll_fn(|cx| {
18949 assert!(open.as_mut().poll(cx).is_pending());
18950 module_rx.poll_recv(cx)
18951 })
18952 .await
18953 .unwrap()
18954 .frame;
18955 handler
18956 .handle_control_frame(&rig.modules[PLEXUS].0, route_bind_ack(bind.header.corr))
18957 .await
18958 .unwrap();
18959 let replies = tokio::time::timeout(Duration::from_secs(2), open)
18960 .await
18961 .unwrap()
18962 .unwrap();
18963 assert_eq!(replies.len(), 1);
18964 assert_eq!(parse_error(&replies[0])["code"], error_codes::SCOPE_CHANGED);
18967 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
18968 while rig.modules.get_mut(PLEXUS).unwrap().1.try_recv().is_ok() {}
18969 let route = rig
18970 .bound(Some(AFT), PLEXUS, Some(rig.selector("s", Some(1))))
18971 .await;
18972 assert!(rig.live(&route));
18973 }
18974
18975 #[tokio::test]
18976 async fn scope_without_flow_id_and_unscoped_routes_admit_a_target_without_flow_capability()
18977 {
18978 let mut rig = rig_with_flow_support(false).await;
18979 rig.sync(vec![session(1)]).await;
18980 let route = rig
18981 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
18982 .await;
18983 assert!(rig.live(&route));
18984 assert_eq!(route.stamp().unwrap().attributes.flow_id, None);
18985 let mut record = session(1);
18986 record.attributes.flow_id = Some("flow:7".to_string());
18987 rig.sync(vec![record]).await;
18988 let unscoped = rig.bound(Some(AFT), OTHER, None).await;
18989 assert!(rig.live(&unscoped));
18990 assert_eq!(unscoped.stamp(), None);
18991 }
18992
18993 #[tokio::test]
18994 async fn a_same_epoch_flow_id_change_bumps_version_and_drains_all_scoped_routes() {
18995 let mut rig = rig().await;
18996 let mut record = session(1);
18997 record.attributes.flow_id = Some("flow:7".to_string());
18998 rig.sync(vec![record.clone()]).await;
18999 let before = rig.forwarding.published_scope_tag(OWNER, "s");
19000 let mut owner_route = rig
19001 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
19002 .await;
19003 let mut carrier_route = rig
19004 .bound(Some(AFT), OTHER, Some(rig_selector("s", Some(1))))
19005 .await;
19006 let mut unscoped = rig.bound(Some(AFT), PLEXUS, None).await;
19007 rig.sync(vec![record.clone()]).await;
19008 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
19009 assert!(rig.live(&owner_route) && owner_route.untouched());
19010 assert!(rig.live(&carrier_route) && carrier_route.untouched());
19011
19012 record.attributes.flow_id = Some("flow:8".to_string());
19013 rig.sync(vec![record]).await;
19014 let after = rig.forwarding.published_scope_tag(OWNER, "s").unwrap();
19015 let before = before.unwrap();
19016 assert_eq!(after.scope_epoch, before.scope_epoch);
19017 assert!(after.version > before.version);
19018 for route in [&mut owner_route, &mut carrier_route] {
19019 assert!(!rig.live(route));
19020 assert_eq!(
19021 route.closed_reason(),
19022 RouteCloseReason::ScopeDelegationChanged
19023 );
19024 }
19025 assert!(rig.live(&unscoped) && unscoped.untouched());
19026 for target in [PLEXUS, OTHER] {
19029 let (_, module_rx) = rig.modules.get_mut(target).unwrap();
19030 let goodbye = module_rx
19031 .try_recv()
19032 .expect("the provider sees the drain")
19033 .frame;
19034 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19035 assert!(module_rx.try_recv().is_err());
19036 }
19037 let rebound = rig
19038 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19039 .await;
19040 assert_eq!(
19041 rebound.stamp().unwrap().attributes.flow_id.as_deref(),
19042 Some("flow:8")
19043 );
19044 }
19045
19046 #[tokio::test]
19051 async fn a_scope_changed_or_ended_between_admission_and_commit_refuses_the_open() {
19052 let mut rig = rig().await;
19053 rig.sync(vec![session(1)]).await;
19054 let mut cotenant = rig
19055 .bound(Some(OWNER), PLEXUS, Some(rig_selector("s", Some(1))))
19056 .await;
19057
19058 let mut changed = session(1);
19059 changed.child_owners.push(Principal::Reserved {
19060 module_id: MAGIC.to_string(),
19061 });
19062 let mut ended = None;
19063 for (code, next) in [
19064 (error_codes::SCOPE_CHANGED, vec![changed]),
19065 (error_codes::SCOPE_ENDED, Vec::new()),
19066 ] {
19067 let relayed = rig
19068 .relayed(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19069 .await;
19070 let (bind_channel, bind_epoch) = route_bind_channel(&relayed.bind);
19071 ended = Some(next.is_empty());
19072 rig.sync(next).await;
19073 rig.ack(&relayed).await;
19074 let replies = relayed.task.await.unwrap();
19075 assert_eq!(replies.len(), 1, "{replies:?}");
19076 assert_eq!(parse_error(&replies[0])["code"], code);
19077 assert_eq!(
19078 subc_protocol::error_codes::is_retryable_route_open(code),
19079 code == error_codes::SCOPE_CHANGED
19080 );
19081 assert_eq!(rig.forwarding.reserved_route_count().unwrap(), (0, 0));
19082 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19084 let mut goodbyes = Vec::new();
19087 while let Ok(outbound) = plexus_rx.try_recv() {
19088 assert_eq!(outbound.frame.header.ty, FrameType::Goodbye);
19089 goodbyes.push((outbound.frame.header.channel, outbound.frame.header.epoch));
19090 }
19091 assert!(
19092 goodbyes.contains(&(bind_channel, bind_epoch)),
19093 "{goodbyes:?}"
19094 );
19095 assert!(rig
19096 .handler
19097 .registry
19098 .get_module_by_connection(rig.modules[PLEXUS].0.connection_id)
19099 .unwrap()
19100 .is_some());
19101 }
19102 assert_eq!(ended, Some(true));
19103 assert_eq!(cotenant.closed_reason(), RouteCloseReason::ScopeEnded);
19106 }
19107
19108 #[tokio::test]
19111 async fn each_revocation_drains_exactly_the_affected_routes_with_its_own_reason() {
19112 struct Case {
19113 name: &'static str,
19114 change: fn(&mut ScopeRecord),
19115 closed: [Option<RouteCloseReason>; 4],
19118 }
19119 use RouteCloseReason::*;
19120 let cases = [
19121 Case {
19122 name: "a carrier entry removed",
19123 change: |r| {
19124 r.carriers.retain(|c| {
19125 c.principal
19126 != Principal::Reserved {
19127 module_id: AFT.to_string(),
19128 }
19129 })
19130 },
19131 closed: [None, Some(ScopeCarrierRemoved), None, None],
19132 },
19133 Case {
19134 name: "a target removed from a carrier",
19135 change: |r| r.carriers[1].targets = Some(vec![PLEXUS.to_string()]),
19136 closed: [None, None, None, Some(ScopeCarrierRemoved)],
19137 },
19138 Case {
19139 name: "a bare carrier narrowed to targets",
19140 change: |r| r.carriers[0].targets = Some(vec![OTHER.to_string()]),
19141 closed: [None, Some(ScopeCarrierRemoved), None, None],
19142 },
19143 Case {
19144 name: "delegates turned off",
19145 change: |r| r.attributes.delegates = false,
19146 closed: [Some(ScopeDelegationChanged); 4],
19147 },
19148 Case {
19149 name: "agent_id changed",
19150 change: |r| r.attributes.agent_id = Some("agent-2".to_string()),
19151 closed: [Some(ScopeDelegationChanged); 4],
19152 },
19153 Case {
19154 name: "a carrier added, child owners changed, the record re-sent",
19155 change: |r| {
19156 r.carriers.push(carrier(MAGIC, None));
19157 r.child_owners.push(Principal::Reserved {
19158 module_id: MAGIC.to_string(),
19159 });
19160 },
19161 closed: [None; 4],
19162 },
19163 Case {
19164 name: "a target added",
19165 change: |r| {
19166 r.carriers[1]
19167 .targets
19168 .as_mut()
19169 .unwrap()
19170 .push("third".to_string())
19171 },
19172 closed: [None; 4],
19173 },
19174 Case {
19175 name: "delegates turned on",
19176 change: |r| r.attributes.delegates = true,
19177 closed: [None; 4],
19178 },
19179 ];
19180 for case in cases {
19181 let mut rig = rig().await;
19182 rig.sync(vec![session(1)]).await;
19183 let scope = || Some(rig_selector("s", Some(1)));
19184 let mut routes = [
19185 rig.bound(Some(OWNER), PLEXUS, scope()).await,
19186 rig.bound(Some(AFT), PLEXUS, scope()).await,
19187 rig.bound(Some(BROCA), PLEXUS, scope()).await,
19188 rig.bound(Some(BROCA), OTHER, scope()).await,
19189 ];
19190 let mut record = session(1);
19191 (case.change)(&mut record);
19192 rig.sync(vec![record]).await;
19193 for (index, expected) in case.closed.iter().enumerate() {
19194 let route = &mut routes[index];
19195 match expected {
19196 Some(reason) => {
19197 assert!(!rig.live(route), "{}: route {index} still live", case.name);
19198 assert_eq!(
19199 route.closed_reason(),
19200 *reason,
19201 "{}: route {index}",
19202 case.name
19203 );
19204 }
19205 None => {
19206 assert!(rig.live(route), "{}: route {index} closed", case.name);
19207 assert!(
19208 route.untouched(),
19209 "{}: route {index} was told something",
19210 case.name
19211 );
19212 }
19213 }
19214 }
19215 }
19216 }
19217
19218 #[tokio::test]
19219 async fn ending_or_replacing_a_scope_and_a_parent_ending_drain_every_route_under_it() {
19220 for next in [Vec::new(), vec![session(2)]] {
19222 let mut rig = rig().await;
19223 rig.sync(vec![session(1)]).await;
19224 let mut route = rig
19225 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19226 .await;
19227 rig.sync(next).await;
19228 assert!(!rig.live(&route));
19229 assert_eq!(route.closed_reason(), RouteCloseReason::ScopeEnded);
19230 }
19231
19232 let mut rig = rig().await;
19235 let mut child = session(1);
19236 child.scope_ref = "child".to_string();
19237 child.kind = ScopeKind::Worker;
19238 child.parent = Some(ScopeParent::new(
19239 Principal::Reserved {
19240 module_id: OWNER.to_string(),
19241 },
19242 "s".to_string(),
19243 1,
19244 ));
19245 rig.sync(vec![session(1), child.clone()]).await;
19246 let mut child_route = rig
19247 .bound(Some(AFT), PLEXUS, Some(rig_selector("child", Some(1))))
19248 .await;
19249 assert_eq!(
19250 child_route.stamp().unwrap().parent_state,
19251 Some(ParentState::Linked)
19252 );
19253 rig.sync(vec![child]).await;
19254 assert!(!rig.live(&child_route));
19255 assert_eq!(
19256 child_route.closed_reason(),
19257 RouteCloseReason::ScopeParentEnded
19258 );
19259 }
19260
19261 #[tokio::test]
19262 async fn re_sending_an_unchanged_record_drains_nothing_and_a_new_carrier_leaves_in_flight_calls(
19263 ) {
19264 let mut rig = rig().await;
19265 rig.sync(vec![session(1)]).await;
19266 let mut route = rig
19267 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19268 .await;
19269 let before = rig.forwarding.published_scope_tag(OWNER, "s");
19270 rig.sync(vec![session(1)]).await;
19271 assert_eq!(rig.forwarding.published_scope_tag(OWNER, "s"), before);
19272 assert!(rig.live(&route) && route.untouched());
19273
19274 let DataRoute::Client(DataRouteState::Bound(binding)) = rig
19278 .forwarding
19279 .lookup_data_route(route.client.connection_id, route.channel, route.epoch)
19280 .unwrap()
19281 else {
19282 panic!("the route is bound");
19283 };
19284 binding.flow.acquire_tagged(9, false).await.unwrap();
19285 let mut widened = session(1);
19286 widened.carriers.push(carrier(MAGIC, None));
19287 rig.sync(vec![widened]).await;
19288 assert!(rig.live(&route) && route.untouched());
19289 let (_, plexus_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19290 assert!(plexus_rx.try_recv().is_err(), "the module is told nothing");
19291 assert_eq!(binding.flow.in_flight(), 1);
19294 binding
19295 .flow
19296 .acquire_tagged(10, false)
19297 .await
19298 .expect("the flow is still open");
19299 }
19300
19301 #[tokio::test]
19304 async fn ending_a_scope_drains_its_routes_on_a_superseded_endpoint() {
19305 let mut rig = rig().await;
19306 rig.sync(vec![session(1)]).await;
19307 let mut on_incumbent = rig
19308 .bound(Some(AFT), PLEXUS, Some(rig_selector("s", Some(1))))
19309 .await;
19310
19311 let (candidate, _candidate_rx) = wide_ctx(9);
19314 let registration = rig
19315 .handler
19316 .registry
19317 .register_candidate_with_control_ops(
19318 manifest(PLEXUS, PROTOCOL_VERSION),
19319 PROTOCOL_VERSION,
19320 candidate.connection_id,
19321 module_baseline_control_ops(),
19322 )
19323 .unwrap();
19324 rig.forwarding
19325 .register_candidate_module_connection(
19326 candidate.connection_id,
19327 PLEXUS.to_string(),
19328 PROTOCOL_VERSION,
19329 manifest_concurrency(®istration.manifest),
19330 candidate.egress.clone(),
19331 )
19332 .unwrap();
19333 rig.forwarding.cutover_candidate(PLEXUS).unwrap().unwrap();
19334 rig.handler
19335 .registry
19336 .promote_candidate(PLEXUS)
19337 .unwrap()
19338 .unwrap();
19339 assert!(rig.live(&on_incumbent), "cutover alone does not drain");
19340
19341 rig.sync(Vec::new()).await;
19342 assert!(!rig.live(&on_incumbent));
19343 assert_eq!(on_incumbent.closed_reason(), RouteCloseReason::ScopeEnded);
19344 let (_, incumbent_rx) = rig.modules.get_mut(PLEXUS).unwrap();
19345 let goodbye = incumbent_rx
19346 .try_recv()
19347 .expect("the superseded endpoint is told")
19348 .frame;
19349 assert_eq!(goodbye.header.ty, FrameType::Goodbye);
19350 }
19351 }
19352}
19353
19354#[cfg(test)]
19355mod concurrency_default_exposure_tests {
19356 use super::*;
19357
19358 fn hello_body(role_json: &str) -> Vec<u8> {
19359 format!(
19360 r#"{{"protocol_ver":2,"module_id":"m","manifest":{{"module_id":"m","module_version":"1.0.0","protocol_ver":2,"trust_tier":"first_party","provides":[{role_json}],"consumes":[],"bindings":{{"storage":{{"kind":"sqlite","scope":"project","owns_schema":false}},"vault_grants":[],"identity":{{"requires":[],"optional":[]}}}}}}}}"#
19361 )
19362 .into_bytes()
19363 }
19364
19365 fn manifest_from(body: &[u8]) -> ModuleManifest {
19366 let value: serde_json::Value = serde_json::from_slice(body).expect("hello parses");
19367 serde_json::from_value(value.get("manifest").expect("manifest key").clone())
19368 .expect("manifest parses")
19369 }
19370
19371 const SURFACE_TAIL: &str = r#""operations":[],"config_schema":{"type":"object"},"observability":[],"identity_scope":[]"#;
19372
19373 #[test]
19374 fn absent_concurrency_on_management_surface_is_reported_as_defaulted() {
19375 let body = hello_body(&format!(
19376 r#"{{"role":"management_surface",{SURFACE_TAIL}}}"#
19377 ));
19378 let manifest = manifest_from(&body);
19379 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
19382 assert!(manifest_concurrency_was_defaulted(&body, &manifest));
19383 }
19384
19385 #[test]
19386 fn declared_concurrency_is_not_reported_even_when_it_equals_the_default() {
19387 let body = hello_body(&format!(
19388 r#"{{"role":"management_surface",{SURFACE_TAIL},"concurrency":"module_managed"}}"#
19389 ));
19390 let manifest = manifest_from(&body);
19391 assert_eq!(manifest_concurrency(&manifest), Concurrency::ModuleManaged);
19392 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19393 }
19394
19395 #[test]
19396 fn non_management_roles_are_never_reported() {
19397 let body = hello_body(
19398 r#"{"role":"internal_service","service_id":"s","transport":"bulk","agent_facing":false,"operations":[]}"#,
19399 );
19400 let manifest = manifest_from(&body);
19401 assert!(!manifest_concurrency_was_defaulted(&body, &manifest));
19402 }
19403}